Model save
Browse files- README.md +77 -0
- adapter_model.safetensors +1 -1
- all_results.json +8 -0
- train_results.json +8 -0
- trainer_state.json +2497 -0
README.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: peft
|
| 3 |
+
tags:
|
| 4 |
+
- trl
|
| 5 |
+
- dpo
|
| 6 |
+
- generated_from_trainer
|
| 7 |
+
base_model: allenai/tulu-2-7b
|
| 8 |
+
model-index:
|
| 9 |
+
- name: tulu2-7b-cost-UF-both-5e-7
|
| 10 |
+
results: []
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
| 14 |
+
should probably proofread and complete it, then remove this comment. -->
|
| 15 |
+
|
| 16 |
+
# tulu2-7b-cost-UF-both-5e-7
|
| 17 |
+
|
| 18 |
+
This model is a fine-tuned version of [allenai/tulu-2-7b](https://huggingface.co/allenai/tulu-2-7b) on the None dataset.
|
| 19 |
+
It achieves the following results on the evaluation set:
|
| 20 |
+
- Loss: 0.6946
|
| 21 |
+
- Rewards/chosen: 0.0316
|
| 22 |
+
- Rewards/rejected: 0.0333
|
| 23 |
+
- Rewards/accuracies: 0.5195
|
| 24 |
+
- Rewards/margins: -0.0018
|
| 25 |
+
- Rewards/margins Max: 0.0952
|
| 26 |
+
- Rewards/margins Min: -0.1041
|
| 27 |
+
- Rewards/margins Std: 0.0646
|
| 28 |
+
- Logps/rejected: -316.1527
|
| 29 |
+
- Logps/chosen: -330.8240
|
| 30 |
+
- Logits/rejected: 0.8900
|
| 31 |
+
- Logits/chosen: 0.7447
|
| 32 |
+
|
| 33 |
+
## Model description
|
| 34 |
+
|
| 35 |
+
More information needed
|
| 36 |
+
|
| 37 |
+
## Intended uses & limitations
|
| 38 |
+
|
| 39 |
+
More information needed
|
| 40 |
+
|
| 41 |
+
## Training and evaluation data
|
| 42 |
+
|
| 43 |
+
More information needed
|
| 44 |
+
|
| 45 |
+
## Training procedure
|
| 46 |
+
|
| 47 |
+
### Training hyperparameters
|
| 48 |
+
|
| 49 |
+
The following hyperparameters were used during training:
|
| 50 |
+
- learning_rate: 5e-07
|
| 51 |
+
- train_batch_size: 2
|
| 52 |
+
- eval_batch_size: 8
|
| 53 |
+
- seed: 42
|
| 54 |
+
- distributed_type: multi-GPU
|
| 55 |
+
- num_devices: 2
|
| 56 |
+
- gradient_accumulation_steps: 4
|
| 57 |
+
- total_train_batch_size: 16
|
| 58 |
+
- total_eval_batch_size: 16
|
| 59 |
+
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
|
| 60 |
+
- lr_scheduler_type: cosine
|
| 61 |
+
- lr_scheduler_warmup_ratio: 0.1
|
| 62 |
+
- num_epochs: 1
|
| 63 |
+
|
| 64 |
+
### Training results
|
| 65 |
+
|
| 66 |
+
| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/margins Max | Rewards/margins Min | Rewards/margins Std | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
|
| 67 |
+
|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:-------------------:|:-------------------:|:-------------------:|:--------------:|:------------:|:---------------:|:-------------:|
|
| 68 |
+
| 0.6506 | 1.0 | 1359 | 0.6946 | 0.0316 | 0.0333 | 0.5195 | -0.0018 | 0.0952 | -0.1041 | 0.0646 | -316.1527 | -330.8240 | 0.8900 | 0.7447 |
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
### Framework versions
|
| 72 |
+
|
| 73 |
+
- PEFT 0.7.1
|
| 74 |
+
- Transformers 4.39.0.dev0
|
| 75 |
+
- Pytorch 2.1.2+cu121
|
| 76 |
+
- Datasets 2.14.6
|
| 77 |
+
- Tokenizers 0.15.2
|
adapter_model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 639692768
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3c28d759a4a3ea60879c44f369c367bcbe75dd001b8cc225b57e953604f92c8e
|
| 3 |
size 639692768
|
all_results.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"epoch": 1.0,
|
| 3 |
+
"train_loss": 0.6604200788417925,
|
| 4 |
+
"train_runtime": 10358.1715,
|
| 5 |
+
"train_samples": 21744,
|
| 6 |
+
"train_samples_per_second": 2.099,
|
| 7 |
+
"train_steps_per_second": 0.131
|
| 8 |
+
}
|
train_results.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"epoch": 1.0,
|
| 3 |
+
"train_loss": 0.6604200788417925,
|
| 4 |
+
"train_runtime": 10358.1715,
|
| 5 |
+
"train_samples": 21744,
|
| 6 |
+
"train_samples_per_second": 2.099,
|
| 7 |
+
"train_steps_per_second": 0.131
|
| 8 |
+
}
|
trainer_state.json
ADDED
|
@@ -0,0 +1,2497 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_metric": null,
|
| 3 |
+
"best_model_checkpoint": null,
|
| 4 |
+
"epoch": 1.0,
|
| 5 |
+
"eval_steps": 100,
|
| 6 |
+
"global_step": 1359,
|
| 7 |
+
"is_hyper_param_search": false,
|
| 8 |
+
"is_local_process_zero": true,
|
| 9 |
+
"is_world_process_zero": true,
|
| 10 |
+
"log_history": [
|
| 11 |
+
{
|
| 12 |
+
"epoch": 0.0,
|
| 13 |
+
"grad_norm": 0.416015625,
|
| 14 |
+
"learning_rate": 3.676470588235294e-09,
|
| 15 |
+
"logits/chosen": 0.3823161721229553,
|
| 16 |
+
"logits/rejected": 0.49768388271331787,
|
| 17 |
+
"logps/chosen": -143.86793518066406,
|
| 18 |
+
"logps/rejected": -156.11575317382812,
|
| 19 |
+
"loss": 0.6931,
|
| 20 |
+
"rewards/accuracies": 0.0,
|
| 21 |
+
"rewards/chosen": 0.0,
|
| 22 |
+
"rewards/margins": 0.0,
|
| 23 |
+
"rewards/margins_max": 0.0,
|
| 24 |
+
"rewards/margins_min": 0.0,
|
| 25 |
+
"rewards/margins_std": 0.0,
|
| 26 |
+
"rewards/rejected": 0.0,
|
| 27 |
+
"step": 1
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"epoch": 0.01,
|
| 31 |
+
"grad_norm": 0.5,
|
| 32 |
+
"learning_rate": 3.676470588235294e-08,
|
| 33 |
+
"logits/chosen": -0.03673278167843819,
|
| 34 |
+
"logits/rejected": 0.7153948545455933,
|
| 35 |
+
"logps/chosen": -280.6231689453125,
|
| 36 |
+
"logps/rejected": -194.65892028808594,
|
| 37 |
+
"loss": 0.6933,
|
| 38 |
+
"rewards/accuracies": 0.3888888955116272,
|
| 39 |
+
"rewards/chosen": -0.0011863290565088391,
|
| 40 |
+
"rewards/margins": -0.00027060293359681964,
|
| 41 |
+
"rewards/margins_max": 0.0016658814856782556,
|
| 42 |
+
"rewards/margins_min": -0.002207087352871895,
|
| 43 |
+
"rewards/margins_std": 0.0027386024594306946,
|
| 44 |
+
"rewards/rejected": -0.0009157261229120195,
|
| 45 |
+
"step": 10
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"epoch": 0.01,
|
| 49 |
+
"grad_norm": 0.37109375,
|
| 50 |
+
"learning_rate": 7.352941176470588e-08,
|
| 51 |
+
"logits/chosen": 0.007032014429569244,
|
| 52 |
+
"logits/rejected": 0.7839561104774475,
|
| 53 |
+
"logps/chosen": -332.548828125,
|
| 54 |
+
"logps/rejected": -255.7633819580078,
|
| 55 |
+
"loss": 0.6932,
|
| 56 |
+
"rewards/accuracies": 0.5249999761581421,
|
| 57 |
+
"rewards/chosen": -0.0009580751066096127,
|
| 58 |
+
"rewards/margins": 0.00016147561836987734,
|
| 59 |
+
"rewards/margins_max": 0.0031394611578434706,
|
| 60 |
+
"rewards/margins_min": -0.0028165101539343596,
|
| 61 |
+
"rewards/margins_std": 0.004211507737636566,
|
| 62 |
+
"rewards/rejected": -0.0011195506667718291,
|
| 63 |
+
"step": 20
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"epoch": 0.02,
|
| 67 |
+
"grad_norm": 0.51171875,
|
| 68 |
+
"learning_rate": 1.1029411764705881e-07,
|
| 69 |
+
"logits/chosen": -0.04039617255330086,
|
| 70 |
+
"logits/rejected": 0.607696533203125,
|
| 71 |
+
"logps/chosen": -272.3553161621094,
|
| 72 |
+
"logps/rejected": -256.77227783203125,
|
| 73 |
+
"loss": 0.6932,
|
| 74 |
+
"rewards/accuracies": 0.4625000059604645,
|
| 75 |
+
"rewards/chosen": -0.000774255720898509,
|
| 76 |
+
"rewards/margins": 8.788631384959444e-05,
|
| 77 |
+
"rewards/margins_max": 0.001997451763600111,
|
| 78 |
+
"rewards/margins_min": -0.0018216792959719896,
|
| 79 |
+
"rewards/margins_std": 0.002700533252209425,
|
| 80 |
+
"rewards/rejected": -0.0008621420711278915,
|
| 81 |
+
"step": 30
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"epoch": 0.03,
|
| 85 |
+
"grad_norm": 0.42578125,
|
| 86 |
+
"learning_rate": 1.4705882352941175e-07,
|
| 87 |
+
"logits/chosen": -0.06750641763210297,
|
| 88 |
+
"logits/rejected": 0.6604307889938354,
|
| 89 |
+
"logps/chosen": -295.84283447265625,
|
| 90 |
+
"logps/rejected": -260.4482421875,
|
| 91 |
+
"loss": 0.693,
|
| 92 |
+
"rewards/accuracies": 0.4625000059604645,
|
| 93 |
+
"rewards/chosen": -0.0007127780700102448,
|
| 94 |
+
"rewards/margins": 0.00020934769418090582,
|
| 95 |
+
"rewards/margins_max": 0.0021955964621156454,
|
| 96 |
+
"rewards/margins_min": -0.0017769013065844774,
|
| 97 |
+
"rewards/margins_std": 0.0028089801780879498,
|
| 98 |
+
"rewards/rejected": -0.0009221257641911507,
|
| 99 |
+
"step": 40
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"epoch": 0.04,
|
| 103 |
+
"grad_norm": 0.470703125,
|
| 104 |
+
"learning_rate": 1.8382352941176472e-07,
|
| 105 |
+
"logits/chosen": -0.015060709789395332,
|
| 106 |
+
"logits/rejected": 0.6939523220062256,
|
| 107 |
+
"logps/chosen": -287.7695617675781,
|
| 108 |
+
"logps/rejected": -237.29794311523438,
|
| 109 |
+
"loss": 0.6927,
|
| 110 |
+
"rewards/accuracies": 0.5375000238418579,
|
| 111 |
+
"rewards/chosen": 0.0005769954295828938,
|
| 112 |
+
"rewards/margins": 0.0007454338483512402,
|
| 113 |
+
"rewards/margins_max": 0.0031318417750298977,
|
| 114 |
+
"rewards/margins_min": -0.0016409741947427392,
|
| 115 |
+
"rewards/margins_std": 0.0033748901914805174,
|
| 116 |
+
"rewards/rejected": -0.0001684384187683463,
|
| 117 |
+
"step": 50
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"epoch": 0.04,
|
| 121 |
+
"grad_norm": 0.49609375,
|
| 122 |
+
"learning_rate": 2.2058823529411763e-07,
|
| 123 |
+
"logits/chosen": -0.0018195391166955233,
|
| 124 |
+
"logits/rejected": 0.7142434120178223,
|
| 125 |
+
"logps/chosen": -287.01068115234375,
|
| 126 |
+
"logps/rejected": -256.10614013671875,
|
| 127 |
+
"loss": 0.6923,
|
| 128 |
+
"rewards/accuracies": 0.625,
|
| 129 |
+
"rewards/chosen": 0.0012248576385900378,
|
| 130 |
+
"rewards/margins": 0.0020126227755099535,
|
| 131 |
+
"rewards/margins_max": 0.004514734260737896,
|
| 132 |
+
"rewards/margins_min": -0.0004894885933026671,
|
| 133 |
+
"rewards/margins_std": 0.0035385198425501585,
|
| 134 |
+
"rewards/rejected": -0.0007877650787122548,
|
| 135 |
+
"step": 60
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"epoch": 0.05,
|
| 139 |
+
"grad_norm": 0.421875,
|
| 140 |
+
"learning_rate": 2.5735294117647057e-07,
|
| 141 |
+
"logits/chosen": 0.11708433926105499,
|
| 142 |
+
"logits/rejected": 0.5532101988792419,
|
| 143 |
+
"logps/chosen": -279.79791259765625,
|
| 144 |
+
"logps/rejected": -243.6463623046875,
|
| 145 |
+
"loss": 0.692,
|
| 146 |
+
"rewards/accuracies": 0.6625000238418579,
|
| 147 |
+
"rewards/chosen": 0.0009990496328100562,
|
| 148 |
+
"rewards/margins": 0.0019516434986144304,
|
| 149 |
+
"rewards/margins_max": 0.004653914365917444,
|
| 150 |
+
"rewards/margins_min": -0.0007506275433115661,
|
| 151 |
+
"rewards/margins_std": 0.0038215883541852236,
|
| 152 |
+
"rewards/rejected": -0.0009525938075967133,
|
| 153 |
+
"step": 70
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"epoch": 0.06,
|
| 157 |
+
"grad_norm": 0.50390625,
|
| 158 |
+
"learning_rate": 2.941176470588235e-07,
|
| 159 |
+
"logits/chosen": -0.1581064760684967,
|
| 160 |
+
"logits/rejected": 0.7092851996421814,
|
| 161 |
+
"logps/chosen": -311.269775390625,
|
| 162 |
+
"logps/rejected": -251.24081420898438,
|
| 163 |
+
"loss": 0.6915,
|
| 164 |
+
"rewards/accuracies": 0.762499988079071,
|
| 165 |
+
"rewards/chosen": 0.002858319552615285,
|
| 166 |
+
"rewards/margins": 0.0039031144697219133,
|
| 167 |
+
"rewards/margins_max": 0.006772955413907766,
|
| 168 |
+
"rewards/margins_min": 0.001033273758366704,
|
| 169 |
+
"rewards/margins_std": 0.004058566875755787,
|
| 170 |
+
"rewards/rejected": -0.001044794567860663,
|
| 171 |
+
"step": 80
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"epoch": 0.07,
|
| 175 |
+
"grad_norm": 0.390625,
|
| 176 |
+
"learning_rate": 3.3088235294117644e-07,
|
| 177 |
+
"logits/chosen": -0.045442212373018265,
|
| 178 |
+
"logits/rejected": 0.7158526182174683,
|
| 179 |
+
"logps/chosen": -265.78192138671875,
|
| 180 |
+
"logps/rejected": -218.0985870361328,
|
| 181 |
+
"loss": 0.691,
|
| 182 |
+
"rewards/accuracies": 0.75,
|
| 183 |
+
"rewards/chosen": 0.003086745273321867,
|
| 184 |
+
"rewards/margins": 0.0036944295279681683,
|
| 185 |
+
"rewards/margins_max": 0.007027274928987026,
|
| 186 |
+
"rewards/margins_min": 0.0003615838068071753,
|
| 187 |
+
"rewards/margins_std": 0.0047133550979197025,
|
| 188 |
+
"rewards/rejected": -0.0006076841382309794,
|
| 189 |
+
"step": 90
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"epoch": 0.07,
|
| 193 |
+
"grad_norm": 0.458984375,
|
| 194 |
+
"learning_rate": 3.6764705882352943e-07,
|
| 195 |
+
"logits/chosen": 0.0736229196190834,
|
| 196 |
+
"logits/rejected": 0.7526043057441711,
|
| 197 |
+
"logps/chosen": -253.97119140625,
|
| 198 |
+
"logps/rejected": -216.62966918945312,
|
| 199 |
+
"loss": 0.6907,
|
| 200 |
+
"rewards/accuracies": 0.762499988079071,
|
| 201 |
+
"rewards/chosen": 0.004654365126043558,
|
| 202 |
+
"rewards/margins": 0.005358897615224123,
|
| 203 |
+
"rewards/margins_max": 0.008878113701939583,
|
| 204 |
+
"rewards/margins_min": 0.0018396815285086632,
|
| 205 |
+
"rewards/margins_std": 0.004976923577487469,
|
| 206 |
+
"rewards/rejected": -0.0007045331294648349,
|
| 207 |
+
"step": 100
|
| 208 |
+
},
|
| 209 |
+
{
|
| 210 |
+
"epoch": 0.08,
|
| 211 |
+
"grad_norm": 0.42578125,
|
| 212 |
+
"learning_rate": 4.044117647058823e-07,
|
| 213 |
+
"logits/chosen": -0.09733302891254425,
|
| 214 |
+
"logits/rejected": 0.6439273953437805,
|
| 215 |
+
"logps/chosen": -289.07769775390625,
|
| 216 |
+
"logps/rejected": -234.5022735595703,
|
| 217 |
+
"loss": 0.6899,
|
| 218 |
+
"rewards/accuracies": 0.800000011920929,
|
| 219 |
+
"rewards/chosen": 0.005760089494287968,
|
| 220 |
+
"rewards/margins": 0.006674544420093298,
|
| 221 |
+
"rewards/margins_max": 0.010509292595088482,
|
| 222 |
+
"rewards/margins_min": 0.002839796245098114,
|
| 223 |
+
"rewards/margins_std": 0.005423153750598431,
|
| 224 |
+
"rewards/rejected": -0.0009144555078819394,
|
| 225 |
+
"step": 110
|
| 226 |
+
},
|
| 227 |
+
{
|
| 228 |
+
"epoch": 0.09,
|
| 229 |
+
"grad_norm": 0.423828125,
|
| 230 |
+
"learning_rate": 4.4117647058823526e-07,
|
| 231 |
+
"logits/chosen": 0.018124500289559364,
|
| 232 |
+
"logits/rejected": 0.7241690754890442,
|
| 233 |
+
"logps/chosen": -311.22100830078125,
|
| 234 |
+
"logps/rejected": -241.0907440185547,
|
| 235 |
+
"loss": 0.6888,
|
| 236 |
+
"rewards/accuracies": 0.8500000238418579,
|
| 237 |
+
"rewards/chosen": 0.008410803973674774,
|
| 238 |
+
"rewards/margins": 0.007946285419166088,
|
| 239 |
+
"rewards/margins_max": 0.013005959801375866,
|
| 240 |
+
"rewards/margins_min": 0.002886611968278885,
|
| 241 |
+
"rewards/margins_std": 0.0071554603055119514,
|
| 242 |
+
"rewards/rejected": 0.000464518612716347,
|
| 243 |
+
"step": 120
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"epoch": 0.1,
|
| 247 |
+
"grad_norm": 0.494140625,
|
| 248 |
+
"learning_rate": 4.779411764705882e-07,
|
| 249 |
+
"logits/chosen": -0.1438770443201065,
|
| 250 |
+
"logits/rejected": 0.6687305569648743,
|
| 251 |
+
"logps/chosen": -265.34161376953125,
|
| 252 |
+
"logps/rejected": -225.00424194335938,
|
| 253 |
+
"loss": 0.6875,
|
| 254 |
+
"rewards/accuracies": 0.862500011920929,
|
| 255 |
+
"rewards/chosen": 0.011373260989785194,
|
| 256 |
+
"rewards/margins": 0.01142856478691101,
|
| 257 |
+
"rewards/margins_max": 0.018687166273593903,
|
| 258 |
+
"rewards/margins_min": 0.004169962834566832,
|
| 259 |
+
"rewards/margins_std": 0.010265212506055832,
|
| 260 |
+
"rewards/rejected": -5.530398993869312e-05,
|
| 261 |
+
"step": 130
|
| 262 |
+
},
|
| 263 |
+
{
|
| 264 |
+
"epoch": 0.1,
|
| 265 |
+
"grad_norm": 0.423828125,
|
| 266 |
+
"learning_rate": 4.999868030671756e-07,
|
| 267 |
+
"logits/chosen": 0.05402200296521187,
|
| 268 |
+
"logits/rejected": 0.6528539657592773,
|
| 269 |
+
"logps/chosen": -292.96112060546875,
|
| 270 |
+
"logps/rejected": -259.50225830078125,
|
| 271 |
+
"loss": 0.6862,
|
| 272 |
+
"rewards/accuracies": 0.9125000238418579,
|
| 273 |
+
"rewards/chosen": 0.013445700518786907,
|
| 274 |
+
"rewards/margins": 0.014345765113830566,
|
| 275 |
+
"rewards/margins_max": 0.022081127390265465,
|
| 276 |
+
"rewards/margins_min": 0.006610400043427944,
|
| 277 |
+
"rewards/margins_std": 0.010939454659819603,
|
| 278 |
+
"rewards/rejected": -0.0009000620921142399,
|
| 279 |
+
"step": 140
|
| 280 |
+
},
|
| 281 |
+
{
|
| 282 |
+
"epoch": 0.11,
|
| 283 |
+
"grad_norm": 0.462890625,
|
| 284 |
+
"learning_rate": 4.998383535732973e-07,
|
| 285 |
+
"logits/chosen": -0.16393637657165527,
|
| 286 |
+
"logits/rejected": 0.45549267530441284,
|
| 287 |
+
"logps/chosen": -290.1100158691406,
|
| 288 |
+
"logps/rejected": -285.8172302246094,
|
| 289 |
+
"loss": 0.6856,
|
| 290 |
+
"rewards/accuracies": 0.925000011920929,
|
| 291 |
+
"rewards/chosen": 0.016459070146083832,
|
| 292 |
+
"rewards/margins": 0.016152694821357727,
|
| 293 |
+
"rewards/margins_max": 0.02362522855401039,
|
| 294 |
+
"rewards/margins_min": 0.008680161088705063,
|
| 295 |
+
"rewards/margins_std": 0.010567759163677692,
|
| 296 |
+
"rewards/rejected": 0.00030637450981885195,
|
| 297 |
+
"step": 150
|
| 298 |
+
},
|
| 299 |
+
{
|
| 300 |
+
"epoch": 0.12,
|
| 301 |
+
"grad_norm": 0.494140625,
|
| 302 |
+
"learning_rate": 4.995250566954361e-07,
|
| 303 |
+
"logits/chosen": 0.018339941278100014,
|
| 304 |
+
"logits/rejected": 0.7964209914207458,
|
| 305 |
+
"logps/chosen": -283.98577880859375,
|
| 306 |
+
"logps/rejected": -241.65353393554688,
|
| 307 |
+
"loss": 0.6842,
|
| 308 |
+
"rewards/accuracies": 0.8999999761581421,
|
| 309 |
+
"rewards/chosen": 0.01847967877984047,
|
| 310 |
+
"rewards/margins": 0.01731337048113346,
|
| 311 |
+
"rewards/margins_max": 0.028514724224805832,
|
| 312 |
+
"rewards/margins_min": 0.006112017668783665,
|
| 313 |
+
"rewards/margins_std": 0.015841105952858925,
|
| 314 |
+
"rewards/rejected": 0.0011663079494610429,
|
| 315 |
+
"step": 160
|
| 316 |
+
},
|
| 317 |
+
{
|
| 318 |
+
"epoch": 0.13,
|
| 319 |
+
"grad_norm": 0.5546875,
|
| 320 |
+
"learning_rate": 4.990471191519357e-07,
|
| 321 |
+
"logits/chosen": 0.03216102719306946,
|
| 322 |
+
"logits/rejected": 0.7764664888381958,
|
| 323 |
+
"logps/chosen": -279.2666931152344,
|
| 324 |
+
"logps/rejected": -237.353515625,
|
| 325 |
+
"loss": 0.6827,
|
| 326 |
+
"rewards/accuracies": 0.925000011920929,
|
| 327 |
+
"rewards/chosen": 0.023826319724321365,
|
| 328 |
+
"rewards/margins": 0.02013733796775341,
|
| 329 |
+
"rewards/margins_max": 0.03089248575270176,
|
| 330 |
+
"rewards/margins_min": 0.009382191114127636,
|
| 331 |
+
"rewards/margins_std": 0.015210075303912163,
|
| 332 |
+
"rewards/rejected": 0.0036889780312776566,
|
| 333 |
+
"step": 170
|
| 334 |
+
},
|
| 335 |
+
{
|
| 336 |
+
"epoch": 0.13,
|
| 337 |
+
"grad_norm": 0.4453125,
|
| 338 |
+
"learning_rate": 4.984048562937129e-07,
|
| 339 |
+
"logits/chosen": -0.034670569002628326,
|
| 340 |
+
"logits/rejected": 0.5347191095352173,
|
| 341 |
+
"logps/chosen": -253.5879364013672,
|
| 342 |
+
"logps/rejected": -252.5656280517578,
|
| 343 |
+
"loss": 0.6827,
|
| 344 |
+
"rewards/accuracies": 0.9375,
|
| 345 |
+
"rewards/chosen": 0.021980373188853264,
|
| 346 |
+
"rewards/margins": 0.020695097744464874,
|
| 347 |
+
"rewards/margins_max": 0.03351113945245743,
|
| 348 |
+
"rewards/margins_min": 0.007879055105149746,
|
| 349 |
+
"rewards/margins_std": 0.018124621361494064,
|
| 350 |
+
"rewards/rejected": 0.0012852762592956424,
|
| 351 |
+
"step": 180
|
| 352 |
+
},
|
| 353 |
+
{
|
| 354 |
+
"epoch": 0.14,
|
| 355 |
+
"grad_norm": 0.37109375,
|
| 356 |
+
"learning_rate": 4.975986918961825e-07,
|
| 357 |
+
"logits/chosen": -0.0010436892043799162,
|
| 358 |
+
"logits/rejected": 0.5325745344161987,
|
| 359 |
+
"logps/chosen": -258.70465087890625,
|
| 360 |
+
"logps/rejected": -254.8517608642578,
|
| 361 |
+
"loss": 0.6811,
|
| 362 |
+
"rewards/accuracies": 0.925000011920929,
|
| 363 |
+
"rewards/chosen": 0.02575596049427986,
|
| 364 |
+
"rewards/margins": 0.0235120952129364,
|
| 365 |
+
"rewards/margins_max": 0.037295907735824585,
|
| 366 |
+
"rewards/margins_min": 0.009728283621370792,
|
| 367 |
+
"rewards/margins_std": 0.019493255764245987,
|
| 368 |
+
"rewards/rejected": 0.0022438622545450926,
|
| 369 |
+
"step": 190
|
| 370 |
+
},
|
| 371 |
+
{
|
| 372 |
+
"epoch": 0.15,
|
| 373 |
+
"grad_norm": 0.447265625,
|
| 374 |
+
"learning_rate": 4.966291578796448e-07,
|
| 375 |
+
"logits/chosen": -0.08659172803163528,
|
| 376 |
+
"logits/rejected": 0.8055790066719055,
|
| 377 |
+
"logps/chosen": -268.54559326171875,
|
| 378 |
+
"logps/rejected": -214.8706512451172,
|
| 379 |
+
"loss": 0.68,
|
| 380 |
+
"rewards/accuracies": 0.949999988079071,
|
| 381 |
+
"rewards/chosen": 0.02814793959259987,
|
| 382 |
+
"rewards/margins": 0.02799884043633938,
|
| 383 |
+
"rewards/margins_max": 0.042513541877269745,
|
| 384 |
+
"rewards/margins_min": 0.013484140858054161,
|
| 385 |
+
"rewards/margins_std": 0.020526882261037827,
|
| 386 |
+
"rewards/rejected": 0.00014909838500898331,
|
| 387 |
+
"step": 200
|
| 388 |
+
},
|
| 389 |
+
{
|
| 390 |
+
"epoch": 0.15,
|
| 391 |
+
"grad_norm": 0.3515625,
|
| 392 |
+
"learning_rate": 4.954968939583149e-07,
|
| 393 |
+
"logits/chosen": -0.15794704854488373,
|
| 394 |
+
"logits/rejected": 0.6090285181999207,
|
| 395 |
+
"logps/chosen": -289.7901306152344,
|
| 396 |
+
"logps/rejected": -238.60183715820312,
|
| 397 |
+
"loss": 0.6789,
|
| 398 |
+
"rewards/accuracies": 0.925000011920929,
|
| 399 |
+
"rewards/chosen": 0.03186321258544922,
|
| 400 |
+
"rewards/margins": 0.03159675747156143,
|
| 401 |
+
"rewards/margins_max": 0.04762989282608032,
|
| 402 |
+
"rewards/margins_min": 0.015563614666461945,
|
| 403 |
+
"rewards/margins_std": 0.022674284875392914,
|
| 404 |
+
"rewards/rejected": 0.0002664585190359503,
|
| 405 |
+
"step": 210
|
| 406 |
+
},
|
| 407 |
+
{
|
| 408 |
+
"epoch": 0.16,
|
| 409 |
+
"grad_norm": 0.44921875,
|
| 410 |
+
"learning_rate": 4.942026472182297e-07,
|
| 411 |
+
"logits/chosen": 0.012926039285957813,
|
| 412 |
+
"logits/rejected": 0.6630520820617676,
|
| 413 |
+
"logps/chosen": -269.59930419921875,
|
| 414 |
+
"logps/rejected": -248.7339324951172,
|
| 415 |
+
"loss": 0.6778,
|
| 416 |
+
"rewards/accuracies": 0.987500011920929,
|
| 417 |
+
"rewards/chosen": 0.0339738167822361,
|
| 418 |
+
"rewards/margins": 0.03334929794073105,
|
| 419 |
+
"rewards/margins_max": 0.04980397969484329,
|
| 420 |
+
"rewards/margins_min": 0.016894619911909103,
|
| 421 |
+
"rewards/margins_std": 0.023270433768630028,
|
| 422 |
+
"rewards/rejected": 0.0006245142430998385,
|
| 423 |
+
"step": 220
|
| 424 |
+
},
|
| 425 |
+
{
|
| 426 |
+
"epoch": 0.17,
|
| 427 |
+
"grad_norm": 0.43359375,
|
| 428 |
+
"learning_rate": 4.92747271624308e-07,
|
| 429 |
+
"logits/chosen": -0.10246638208627701,
|
| 430 |
+
"logits/rejected": 0.8088499903678894,
|
| 431 |
+
"logps/chosen": -293.9709167480469,
|
| 432 |
+
"logps/rejected": -229.0161895751953,
|
| 433 |
+
"loss": 0.6776,
|
| 434 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 435 |
+
"rewards/chosen": 0.03623218461871147,
|
| 436 |
+
"rewards/margins": 0.03605912998318672,
|
| 437 |
+
"rewards/margins_max": 0.05394510179758072,
|
| 438 |
+
"rewards/margins_min": 0.018173154443502426,
|
| 439 |
+
"rewards/margins_std": 0.025294587016105652,
|
| 440 |
+
"rewards/rejected": 0.0001730593794491142,
|
| 441 |
+
"step": 230
|
| 442 |
+
},
|
| 443 |
+
{
|
| 444 |
+
"epoch": 0.18,
|
| 445 |
+
"grad_norm": 0.50390625,
|
| 446 |
+
"learning_rate": 4.911317274568909e-07,
|
| 447 |
+
"logits/chosen": -0.1043270006775856,
|
| 448 |
+
"logits/rejected": 0.7763813138008118,
|
| 449 |
+
"logps/chosen": -290.52032470703125,
|
| 450 |
+
"logps/rejected": -238.03591918945312,
|
| 451 |
+
"loss": 0.6748,
|
| 452 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 453 |
+
"rewards/chosen": 0.04272182658314705,
|
| 454 |
+
"rewards/margins": 0.04140879958868027,
|
| 455 |
+
"rewards/margins_max": 0.0613585002720356,
|
| 456 |
+
"rewards/margins_min": 0.021459100767970085,
|
| 457 |
+
"rewards/margins_std": 0.0282131377607584,
|
| 458 |
+
"rewards/rejected": 0.0013130264123901725,
|
| 459 |
+
"step": 240
|
| 460 |
+
},
|
| 461 |
+
{
|
| 462 |
+
"epoch": 0.18,
|
| 463 |
+
"grad_norm": 0.4765625,
|
| 464 |
+
"learning_rate": 4.89357080678133e-07,
|
| 465 |
+
"logits/chosen": 0.042627763003110886,
|
| 466 |
+
"logits/rejected": 0.5050796270370483,
|
| 467 |
+
"logps/chosen": -225.12393188476562,
|
| 468 |
+
"logps/rejected": -208.74288940429688,
|
| 469 |
+
"loss": 0.6767,
|
| 470 |
+
"rewards/accuracies": 0.875,
|
| 471 |
+
"rewards/chosen": 0.02972862496972084,
|
| 472 |
+
"rewards/margins": 0.026549100875854492,
|
| 473 |
+
"rewards/margins_max": 0.04413127154111862,
|
| 474 |
+
"rewards/margins_min": 0.008966929279267788,
|
| 475 |
+
"rewards/margins_std": 0.02486494556069374,
|
| 476 |
+
"rewards/rejected": 0.00317952549085021,
|
| 477 |
+
"step": 250
|
| 478 |
+
},
|
| 479 |
+
{
|
| 480 |
+
"epoch": 0.19,
|
| 481 |
+
"grad_norm": 0.6015625,
|
| 482 |
+
"learning_rate": 4.874245022286637e-07,
|
| 483 |
+
"logits/chosen": 0.061648208647966385,
|
| 484 |
+
"logits/rejected": 0.6458357572555542,
|
| 485 |
+
"logps/chosen": -262.064208984375,
|
| 486 |
+
"logps/rejected": -268.61444091796875,
|
| 487 |
+
"loss": 0.677,
|
| 488 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 489 |
+
"rewards/chosen": 0.03966163843870163,
|
| 490 |
+
"rewards/margins": 0.036633603274822235,
|
| 491 |
+
"rewards/margins_max": 0.05438825488090515,
|
| 492 |
+
"rewards/margins_min": 0.01887894608080387,
|
| 493 |
+
"rewards/margins_std": 0.02510887011885643,
|
| 494 |
+
"rewards/rejected": 0.0030280412174761295,
|
| 495 |
+
"step": 260
|
| 496 |
+
},
|
| 497 |
+
{
|
| 498 |
+
"epoch": 0.2,
|
| 499 |
+
"grad_norm": 0.3984375,
|
| 500 |
+
"learning_rate": 4.853352672549815e-07,
|
| 501 |
+
"logits/chosen": 0.05668522045016289,
|
| 502 |
+
"logits/rejected": 0.7193032503128052,
|
| 503 |
+
"logps/chosen": -261.01708984375,
|
| 504 |
+
"logps/rejected": -228.1970672607422,
|
| 505 |
+
"loss": 0.6732,
|
| 506 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 507 |
+
"rewards/chosen": 0.038796816021203995,
|
| 508 |
+
"rewards/margins": 0.038500867784023285,
|
| 509 |
+
"rewards/margins_max": 0.05987783148884773,
|
| 510 |
+
"rewards/margins_min": 0.01712390035390854,
|
| 511 |
+
"rewards/margins_std": 0.030231595039367676,
|
| 512 |
+
"rewards/rejected": 0.0002959508856292814,
|
| 513 |
+
"step": 270
|
| 514 |
+
},
|
| 515 |
+
{
|
| 516 |
+
"epoch": 0.21,
|
| 517 |
+
"grad_norm": 0.3984375,
|
| 518 |
+
"learning_rate": 4.830907542680918e-07,
|
| 519 |
+
"logits/chosen": -0.022307882085442543,
|
| 520 |
+
"logits/rejected": 0.5763358473777771,
|
| 521 |
+
"logps/chosen": -259.81414794921875,
|
| 522 |
+
"logps/rejected": -243.997802734375,
|
| 523 |
+
"loss": 0.6746,
|
| 524 |
+
"rewards/accuracies": 0.9375,
|
| 525 |
+
"rewards/chosen": 0.04487748071551323,
|
| 526 |
+
"rewards/margins": 0.04191521182656288,
|
| 527 |
+
"rewards/margins_max": 0.06111626699566841,
|
| 528 |
+
"rewards/margins_min": 0.022714145481586456,
|
| 529 |
+
"rewards/margins_std": 0.027154404670000076,
|
| 530 |
+
"rewards/rejected": 0.0029622733127325773,
|
| 531 |
+
"step": 280
|
| 532 |
+
},
|
| 533 |
+
{
|
| 534 |
+
"epoch": 0.21,
|
| 535 |
+
"grad_norm": 0.337890625,
|
| 536 |
+
"learning_rate": 4.806924442339425e-07,
|
| 537 |
+
"logits/chosen": -0.14441627264022827,
|
| 538 |
+
"logits/rejected": 0.5065394639968872,
|
| 539 |
+
"logps/chosen": -243.6468505859375,
|
| 540 |
+
"logps/rejected": -224.4588165283203,
|
| 541 |
+
"loss": 0.6735,
|
| 542 |
+
"rewards/accuracies": 0.9125000238418579,
|
| 543 |
+
"rewards/chosen": 0.045001424849033356,
|
| 544 |
+
"rewards/margins": 0.04354133456945419,
|
| 545 |
+
"rewards/margins_max": 0.062143802642822266,
|
| 546 |
+
"rewards/margins_min": 0.02493886649608612,
|
| 547 |
+
"rewards/margins_std": 0.026307862251996994,
|
| 548 |
+
"rewards/rejected": 0.0014600924914702773,
|
| 549 |
+
"step": 290
|
| 550 |
+
},
|
| 551 |
+
{
|
| 552 |
+
"epoch": 0.22,
|
| 553 |
+
"grad_norm": 0.4140625,
|
| 554 |
+
"learning_rate": 4.781419195962598e-07,
|
| 555 |
+
"logits/chosen": -0.09291721135377884,
|
| 556 |
+
"logits/rejected": 0.8421966433525085,
|
| 557 |
+
"logps/chosen": -294.4910583496094,
|
| 558 |
+
"logps/rejected": -235.1195526123047,
|
| 559 |
+
"loss": 0.6669,
|
| 560 |
+
"rewards/accuracies": 0.9375,
|
| 561 |
+
"rewards/chosen": 0.05570490285754204,
|
| 562 |
+
"rewards/margins": 0.05652566999197006,
|
| 563 |
+
"rewards/margins_max": 0.0832170695066452,
|
| 564 |
+
"rewards/margins_min": 0.02983427606523037,
|
| 565 |
+
"rewards/margins_std": 0.0377473346889019,
|
| 566 |
+
"rewards/rejected": -0.0008207682403735816,
|
| 567 |
+
"step": 300
|
| 568 |
+
},
|
| 569 |
+
{
|
| 570 |
+
"epoch": 0.23,
|
| 571 |
+
"grad_norm": 0.427734375,
|
| 572 |
+
"learning_rate": 4.754408632324253e-07,
|
| 573 |
+
"logits/chosen": 0.08360351622104645,
|
| 574 |
+
"logits/rejected": 0.9180965423583984,
|
| 575 |
+
"logps/chosen": -274.46221923828125,
|
| 576 |
+
"logps/rejected": -218.58261108398438,
|
| 577 |
+
"loss": 0.6709,
|
| 578 |
+
"rewards/accuracies": 0.925000011920929,
|
| 579 |
+
"rewards/chosen": 0.04704751446843147,
|
| 580 |
+
"rewards/margins": 0.04507958143949509,
|
| 581 |
+
"rewards/margins_max": 0.07067537307739258,
|
| 582 |
+
"rewards/margins_min": 0.019483786076307297,
|
| 583 |
+
"rewards/margins_std": 0.03619792312383652,
|
| 584 |
+
"rewards/rejected": 0.0019679281394928694,
|
| 585 |
+
"step": 310
|
| 586 |
+
},
|
| 587 |
+
{
|
| 588 |
+
"epoch": 0.24,
|
| 589 |
+
"grad_norm": 0.392578125,
|
| 590 |
+
"learning_rate": 4.725910573430866e-07,
|
| 591 |
+
"logits/chosen": -0.03652850538492203,
|
| 592 |
+
"logits/rejected": 0.7102953195571899,
|
| 593 |
+
"logps/chosen": -259.0872497558594,
|
| 594 |
+
"logps/rejected": -220.06640625,
|
| 595 |
+
"loss": 0.6666,
|
| 596 |
+
"rewards/accuracies": 0.9375,
|
| 597 |
+
"rewards/chosen": 0.052086759358644485,
|
| 598 |
+
"rewards/margins": 0.05210639908909798,
|
| 599 |
+
"rewards/margins_max": 0.07583759725093842,
|
| 600 |
+
"rewards/margins_min": 0.02837519347667694,
|
| 601 |
+
"rewards/margins_std": 0.033560991287231445,
|
| 602 |
+
"rewards/rejected": -1.9636423530755565e-05,
|
| 603 |
+
"step": 320
|
| 604 |
+
},
|
| 605 |
+
{
|
| 606 |
+
"epoch": 0.24,
|
| 607 |
+
"grad_norm": 0.482421875,
|
| 608 |
+
"learning_rate": 4.6959438227623293e-07,
|
| 609 |
+
"logits/chosen": -0.07948180288076401,
|
| 610 |
+
"logits/rejected": 0.6266988515853882,
|
| 611 |
+
"logps/chosen": -291.2080078125,
|
| 612 |
+
"logps/rejected": -269.5301818847656,
|
| 613 |
+
"loss": 0.6677,
|
| 614 |
+
"rewards/accuracies": 0.987500011920929,
|
| 615 |
+
"rewards/chosen": 0.04915199428796768,
|
| 616 |
+
"rewards/margins": 0.050243012607097626,
|
| 617 |
+
"rewards/margins_max": 0.07468355447053909,
|
| 618 |
+
"rewards/margins_min": 0.025802483782172203,
|
| 619 |
+
"rewards/margins_std": 0.03456413000822067,
|
| 620 |
+
"rewards/rejected": -0.0010910223936662078,
|
| 621 |
+
"step": 330
|
| 622 |
+
},
|
| 623 |
+
{
|
| 624 |
+
"epoch": 0.25,
|
| 625 |
+
"grad_norm": 0.451171875,
|
| 626 |
+
"learning_rate": 4.664528152865105e-07,
|
| 627 |
+
"logits/chosen": 0.015779225155711174,
|
| 628 |
+
"logits/rejected": 0.6966221332550049,
|
| 629 |
+
"logps/chosen": -291.6715087890625,
|
| 630 |
+
"logps/rejected": -260.85784912109375,
|
| 631 |
+
"loss": 0.6643,
|
| 632 |
+
"rewards/accuracies": 0.9375,
|
| 633 |
+
"rewards/chosen": 0.06338638812303543,
|
| 634 |
+
"rewards/margins": 0.06281675398349762,
|
| 635 |
+
"rewards/margins_max": 0.09652182459831238,
|
| 636 |
+
"rewards/margins_min": 0.029111674055457115,
|
| 637 |
+
"rewards/margins_std": 0.04766617342829704,
|
| 638 |
+
"rewards/rejected": 0.0005696387379430234,
|
| 639 |
+
"step": 340
|
| 640 |
+
},
|
| 641 |
+
{
|
| 642 |
+
"epoch": 0.26,
|
| 643 |
+
"grad_norm": 0.56640625,
|
| 644 |
+
"learning_rate": 4.6316842923059816e-07,
|
| 645 |
+
"logits/chosen": -0.15278920531272888,
|
| 646 |
+
"logits/rejected": 0.6084717512130737,
|
| 647 |
+
"logps/chosen": -247.9635009765625,
|
| 648 |
+
"logps/rejected": -236.7021484375,
|
| 649 |
+
"loss": 0.6618,
|
| 650 |
+
"rewards/accuracies": 0.987500011920929,
|
| 651 |
+
"rewards/chosen": 0.07176478952169418,
|
| 652 |
+
"rewards/margins": 0.07197294384241104,
|
| 653 |
+
"rewards/margins_max": 0.09642891585826874,
|
| 654 |
+
"rewards/margins_min": 0.047516968101263046,
|
| 655 |
+
"rewards/margins_std": 0.03458596393465996,
|
| 656 |
+
"rewards/rejected": -0.0002081508719129488,
|
| 657 |
+
"step": 350
|
| 658 |
+
},
|
| 659 |
+
{
|
| 660 |
+
"epoch": 0.26,
|
| 661 |
+
"grad_norm": 0.45703125,
|
| 662 |
+
"learning_rate": 4.5974339119950334e-07,
|
| 663 |
+
"logits/chosen": -0.0013063043588772416,
|
| 664 |
+
"logits/rejected": 0.6344822645187378,
|
| 665 |
+
"logps/chosen": -263.31610107421875,
|
| 666 |
+
"logps/rejected": -204.03660583496094,
|
| 667 |
+
"loss": 0.6657,
|
| 668 |
+
"rewards/accuracies": 0.8999999761581421,
|
| 669 |
+
"rewards/chosen": 0.052458666265010834,
|
| 670 |
+
"rewards/margins": 0.04967236518859863,
|
| 671 |
+
"rewards/margins_max": 0.07377265393733978,
|
| 672 |
+
"rewards/margins_min": 0.025572070851922035,
|
| 673 |
+
"rewards/margins_std": 0.034082964062690735,
|
| 674 |
+
"rewards/rejected": 0.0027863027062267065,
|
| 675 |
+
"step": 360
|
| 676 |
+
},
|
| 677 |
+
{
|
| 678 |
+
"epoch": 0.27,
|
| 679 |
+
"grad_norm": 0.451171875,
|
| 680 |
+
"learning_rate": 4.5617996108867997e-07,
|
| 681 |
+
"logits/chosen": -0.09778765588998795,
|
| 682 |
+
"logits/rejected": 0.8527771234512329,
|
| 683 |
+
"logps/chosen": -272.1876525878906,
|
| 684 |
+
"logps/rejected": -224.5320281982422,
|
| 685 |
+
"loss": 0.6615,
|
| 686 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 687 |
+
"rewards/chosen": 0.06909666955471039,
|
| 688 |
+
"rewards/margins": 0.06635682284832001,
|
| 689 |
+
"rewards/margins_max": 0.09376729279756546,
|
| 690 |
+
"rewards/margins_min": 0.03894636407494545,
|
| 691 |
+
"rewards/margins_std": 0.03876424580812454,
|
| 692 |
+
"rewards/rejected": 0.0027398415841162205,
|
| 693 |
+
"step": 370
|
| 694 |
+
},
|
| 695 |
+
{
|
| 696 |
+
"epoch": 0.28,
|
| 697 |
+
"grad_norm": 0.39453125,
|
| 698 |
+
"learning_rate": 4.5248049010691304e-07,
|
| 699 |
+
"logits/chosen": -0.04781482741236687,
|
| 700 |
+
"logits/rejected": 0.6669384241104126,
|
| 701 |
+
"logps/chosen": -270.28253173828125,
|
| 702 |
+
"logps/rejected": -231.1532440185547,
|
| 703 |
+
"loss": 0.6605,
|
| 704 |
+
"rewards/accuracies": 0.949999988079071,
|
| 705 |
+
"rewards/chosen": 0.06150456517934799,
|
| 706 |
+
"rewards/margins": 0.0636376217007637,
|
| 707 |
+
"rewards/margins_max": 0.09591880440711975,
|
| 708 |
+
"rewards/margins_min": 0.03135643154382706,
|
| 709 |
+
"rewards/margins_std": 0.04565249755978584,
|
| 710 |
+
"rewards/rejected": -0.0021330583840608597,
|
| 711 |
+
"step": 380
|
| 712 |
+
},
|
| 713 |
+
{
|
| 714 |
+
"epoch": 0.29,
|
| 715 |
+
"grad_norm": 0.5546875,
|
| 716 |
+
"learning_rate": 4.486474192249533e-07,
|
| 717 |
+
"logits/chosen": 0.034309376031160355,
|
| 718 |
+
"logits/rejected": 0.7420383095741272,
|
| 719 |
+
"logps/chosen": -246.19668579101562,
|
| 720 |
+
"logps/rejected": -215.16323852539062,
|
| 721 |
+
"loss": 0.6629,
|
| 722 |
+
"rewards/accuracies": 0.949999988079071,
|
| 723 |
+
"rewards/chosen": 0.056318044662475586,
|
| 724 |
+
"rewards/margins": 0.06057443469762802,
|
| 725 |
+
"rewards/margins_max": 0.08743326365947723,
|
| 726 |
+
"rewards/margins_min": 0.03371559828519821,
|
| 727 |
+
"rewards/margins_std": 0.037984125316143036,
|
| 728 |
+
"rewards/rejected": -0.0042563872411847115,
|
| 729 |
+
"step": 390
|
| 730 |
+
},
|
| 731 |
+
{
|
| 732 |
+
"epoch": 0.29,
|
| 733 |
+
"grad_norm": 0.5390625,
|
| 734 |
+
"learning_rate": 4.4468327756492504e-07,
|
| 735 |
+
"logits/chosen": 0.09597498923540115,
|
| 736 |
+
"logits/rejected": 0.7928622961044312,
|
| 737 |
+
"logps/chosen": -275.2891540527344,
|
| 738 |
+
"logps/rejected": -246.2664794921875,
|
| 739 |
+
"loss": 0.6631,
|
| 740 |
+
"rewards/accuracies": 0.949999988079071,
|
| 741 |
+
"rewards/chosen": 0.06533468514680862,
|
| 742 |
+
"rewards/margins": 0.06880095601081848,
|
| 743 |
+
"rewards/margins_max": 0.1037551760673523,
|
| 744 |
+
"rewards/margins_min": 0.03384672850370407,
|
| 745 |
+
"rewards/margins_std": 0.04943273216485977,
|
| 746 |
+
"rewards/rejected": -0.0034662713296711445,
|
| 747 |
+
"step": 400
|
| 748 |
+
},
|
| 749 |
+
{
|
| 750 |
+
"epoch": 0.3,
|
| 751 |
+
"grad_norm": 0.4765625,
|
| 752 |
+
"learning_rate": 4.405906807315705e-07,
|
| 753 |
+
"logits/chosen": -0.021347884088754654,
|
| 754 |
+
"logits/rejected": 0.6999279260635376,
|
| 755 |
+
"logps/chosen": -227.08447265625,
|
| 756 |
+
"logps/rejected": -222.6078643798828,
|
| 757 |
+
"loss": 0.6565,
|
| 758 |
+
"rewards/accuracies": 0.949999988079071,
|
| 759 |
+
"rewards/chosen": 0.0646035447716713,
|
| 760 |
+
"rewards/margins": 0.0676642507314682,
|
| 761 |
+
"rewards/margins_max": 0.09568590670824051,
|
| 762 |
+
"rewards/margins_min": 0.039642591029405594,
|
| 763 |
+
"rewards/margins_std": 0.039628610014915466,
|
| 764 |
+
"rewards/rejected": -0.003060706425458193,
|
| 765 |
+
"step": 410
|
| 766 |
+
},
|
| 767 |
+
{
|
| 768 |
+
"epoch": 0.31,
|
| 769 |
+
"grad_norm": 0.41796875,
|
| 770 |
+
"learning_rate": 4.363723290864314e-07,
|
| 771 |
+
"logits/chosen": 0.11118383705615997,
|
| 772 |
+
"logits/rejected": 0.9948932528495789,
|
| 773 |
+
"logps/chosen": -271.27496337890625,
|
| 774 |
+
"logps/rejected": -222.7627410888672,
|
| 775 |
+
"loss": 0.6589,
|
| 776 |
+
"rewards/accuracies": 0.949999988079071,
|
| 777 |
+
"rewards/chosen": 0.07119729369878769,
|
| 778 |
+
"rewards/margins": 0.07638658583164215,
|
| 779 |
+
"rewards/margins_max": 0.120518758893013,
|
| 780 |
+
"rewards/margins_min": 0.032254405319690704,
|
| 781 |
+
"rewards/margins_std": 0.0624123141169548,
|
| 782 |
+
"rewards/rejected": -0.005189282353967428,
|
| 783 |
+
"step": 420
|
| 784 |
+
},
|
| 785 |
+
{
|
| 786 |
+
"epoch": 0.32,
|
| 787 |
+
"grad_norm": 0.486328125,
|
| 788 |
+
"learning_rate": 4.3203100596610723e-07,
|
| 789 |
+
"logits/chosen": -0.016591979190707207,
|
| 790 |
+
"logits/rejected": 0.813875675201416,
|
| 791 |
+
"logps/chosen": -300.334716796875,
|
| 792 |
+
"logps/rejected": -259.6979064941406,
|
| 793 |
+
"loss": 0.656,
|
| 794 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 795 |
+
"rewards/chosen": 0.07723168283700943,
|
| 796 |
+
"rewards/margins": 0.0816957950592041,
|
| 797 |
+
"rewards/margins_max": 0.1194806843996048,
|
| 798 |
+
"rewards/margins_min": 0.043910913169384,
|
| 799 |
+
"rewards/margins_std": 0.05343589186668396,
|
| 800 |
+
"rewards/rejected": -0.004464106634259224,
|
| 801 |
+
"step": 430
|
| 802 |
+
},
|
| 803 |
+
{
|
| 804 |
+
"epoch": 0.32,
|
| 805 |
+
"grad_norm": 0.384765625,
|
| 806 |
+
"learning_rate": 4.2756957584576436e-07,
|
| 807 |
+
"logits/chosen": 0.051903001964092255,
|
| 808 |
+
"logits/rejected": 0.7332680821418762,
|
| 809 |
+
"logps/chosen": -291.6619873046875,
|
| 810 |
+
"logps/rejected": -264.76763916015625,
|
| 811 |
+
"loss": 0.6551,
|
| 812 |
+
"rewards/accuracies": 0.949999988079071,
|
| 813 |
+
"rewards/chosen": 0.07283146679401398,
|
| 814 |
+
"rewards/margins": 0.07750260829925537,
|
| 815 |
+
"rewards/margins_max": 0.11894078552722931,
|
| 816 |
+
"rewards/margins_min": 0.03606444224715233,
|
| 817 |
+
"rewards/margins_std": 0.05860241502523422,
|
| 818 |
+
"rewards/rejected": -0.004671142902225256,
|
| 819 |
+
"step": 440
|
| 820 |
+
},
|
| 821 |
+
{
|
| 822 |
+
"epoch": 0.33,
|
| 823 |
+
"grad_norm": 0.38671875,
|
| 824 |
+
"learning_rate": 4.22990982449109e-07,
|
| 825 |
+
"logits/chosen": 0.00915027130395174,
|
| 826 |
+
"logits/rejected": 0.7316643595695496,
|
| 827 |
+
"logps/chosen": -236.2955780029297,
|
| 828 |
+
"logps/rejected": -220.3606414794922,
|
| 829 |
+
"loss": 0.6569,
|
| 830 |
+
"rewards/accuracies": 0.9375,
|
| 831 |
+
"rewards/chosen": 0.06984328478574753,
|
| 832 |
+
"rewards/margins": 0.07647494226694107,
|
| 833 |
+
"rewards/margins_max": 0.11472654342651367,
|
| 834 |
+
"rewards/margins_min": 0.038223348557949066,
|
| 835 |
+
"rewards/margins_std": 0.05409592390060425,
|
| 836 |
+
"rewards/rejected": -0.006631660275161266,
|
| 837 |
+
"step": 450
|
| 838 |
+
},
|
| 839 |
+
{
|
| 840 |
+
"epoch": 0.34,
|
| 841 |
+
"grad_norm": 0.412109375,
|
| 842 |
+
"learning_rate": 4.1829824680607104e-07,
|
| 843 |
+
"logits/chosen": -0.07657341659069061,
|
| 844 |
+
"logits/rejected": 0.6622528433799744,
|
| 845 |
+
"logps/chosen": -279.7924499511719,
|
| 846 |
+
"logps/rejected": -269.6399230957031,
|
| 847 |
+
"loss": 0.6549,
|
| 848 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 849 |
+
"rewards/chosen": 0.07170796394348145,
|
| 850 |
+
"rewards/margins": 0.0784129947423935,
|
| 851 |
+
"rewards/margins_max": 0.11032243072986603,
|
| 852 |
+
"rewards/margins_min": 0.046503566205501556,
|
| 853 |
+
"rewards/margins_std": 0.045126743614673615,
|
| 854 |
+
"rewards/rejected": -0.0067050280049443245,
|
| 855 |
+
"step": 460
|
| 856 |
+
},
|
| 857 |
+
{
|
| 858 |
+
"epoch": 0.35,
|
| 859 |
+
"grad_norm": 0.453125,
|
| 860 |
+
"learning_rate": 4.134944652594794e-07,
|
| 861 |
+
"logits/chosen": 0.0630350112915039,
|
| 862 |
+
"logits/rejected": 0.6474081873893738,
|
| 863 |
+
"logps/chosen": -261.22418212890625,
|
| 864 |
+
"logps/rejected": -239.87588500976562,
|
| 865 |
+
"loss": 0.6581,
|
| 866 |
+
"rewards/accuracies": 0.949999988079071,
|
| 867 |
+
"rewards/chosen": 0.05761648342013359,
|
| 868 |
+
"rewards/margins": 0.06358405947685242,
|
| 869 |
+
"rewards/margins_max": 0.09130324423313141,
|
| 870 |
+
"rewards/margins_min": 0.03586486726999283,
|
| 871 |
+
"rewards/margins_std": 0.03920084983110428,
|
| 872 |
+
"rewards/rejected": -0.0059675718657672405,
|
| 873 |
+
"step": 470
|
| 874 |
+
},
|
| 875 |
+
{
|
| 876 |
+
"epoch": 0.35,
|
| 877 |
+
"grad_norm": 0.392578125,
|
| 878 |
+
"learning_rate": 4.085828074220451e-07,
|
| 879 |
+
"logits/chosen": -0.19398275017738342,
|
| 880 |
+
"logits/rejected": 0.5935221910476685,
|
| 881 |
+
"logps/chosen": -270.58892822265625,
|
| 882 |
+
"logps/rejected": -245.28076171875,
|
| 883 |
+
"loss": 0.6561,
|
| 884 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 885 |
+
"rewards/chosen": 0.06964448094367981,
|
| 886 |
+
"rewards/margins": 0.08161848783493042,
|
| 887 |
+
"rewards/margins_max": 0.12005958706140518,
|
| 888 |
+
"rewards/margins_min": 0.04317738488316536,
|
| 889 |
+
"rewards/margins_std": 0.0543639250099659,
|
| 890 |
+
"rewards/rejected": -0.011974002234637737,
|
| 891 |
+
"step": 480
|
| 892 |
+
},
|
| 893 |
+
{
|
| 894 |
+
"epoch": 0.36,
|
| 895 |
+
"grad_norm": 0.373046875,
|
| 896 |
+
"learning_rate": 4.035665140849994e-07,
|
| 897 |
+
"logits/chosen": -0.14574551582336426,
|
| 898 |
+
"logits/rejected": 0.5638002157211304,
|
| 899 |
+
"logps/chosen": -274.3948669433594,
|
| 900 |
+
"logps/rejected": -228.3678436279297,
|
| 901 |
+
"loss": 0.6602,
|
| 902 |
+
"rewards/accuracies": 0.949999988079071,
|
| 903 |
+
"rewards/chosen": 0.06451380997896194,
|
| 904 |
+
"rewards/margins": 0.07035050541162491,
|
| 905 |
+
"rewards/margins_max": 0.10534896701574326,
|
| 906 |
+
"rewards/margins_min": 0.035352036356925964,
|
| 907 |
+
"rewards/margins_std": 0.04949530214071274,
|
| 908 |
+
"rewards/rejected": -0.005836695898324251,
|
| 909 |
+
"step": 490
|
| 910 |
+
},
|
| 911 |
+
{
|
| 912 |
+
"epoch": 0.37,
|
| 913 |
+
"grad_norm": 0.470703125,
|
| 914 |
+
"learning_rate": 3.984488950797678e-07,
|
| 915 |
+
"logits/chosen": -0.040041010826826096,
|
| 916 |
+
"logits/rejected": 0.7338499426841736,
|
| 917 |
+
"logps/chosen": -265.7203063964844,
|
| 918 |
+
"logps/rejected": -230.03170776367188,
|
| 919 |
+
"loss": 0.6557,
|
| 920 |
+
"rewards/accuracies": 0.949999988079071,
|
| 921 |
+
"rewards/chosen": 0.07825618237257004,
|
| 922 |
+
"rewards/margins": 0.07796762883663177,
|
| 923 |
+
"rewards/margins_max": 0.11045694351196289,
|
| 924 |
+
"rewards/margins_min": 0.04547831788659096,
|
| 925 |
+
"rewards/margins_std": 0.04594682529568672,
|
| 926 |
+
"rewards/rejected": 0.00028855842538177967,
|
| 927 |
+
"step": 500
|
| 928 |
+
},
|
| 929 |
+
{
|
| 930 |
+
"epoch": 0.38,
|
| 931 |
+
"grad_norm": 0.46484375,
|
| 932 |
+
"learning_rate": 3.9323332709408904e-07,
|
| 933 |
+
"logits/chosen": 0.02443014457821846,
|
| 934 |
+
"logits/rejected": 0.604122519493103,
|
| 935 |
+
"logps/chosen": -254.78701782226562,
|
| 936 |
+
"logps/rejected": -233.02084350585938,
|
| 937 |
+
"loss": 0.6547,
|
| 938 |
+
"rewards/accuracies": 0.887499988079071,
|
| 939 |
+
"rewards/chosen": 0.060421328991651535,
|
| 940 |
+
"rewards/margins": 0.07197172939777374,
|
| 941 |
+
"rewards/margins_max": 0.11490993201732635,
|
| 942 |
+
"rewards/margins_min": 0.029033523052930832,
|
| 943 |
+
"rewards/margins_std": 0.060723792761564255,
|
| 944 |
+
"rewards/rejected": -0.01155039481818676,
|
| 945 |
+
"step": 510
|
| 946 |
+
},
|
| 947 |
+
{
|
| 948 |
+
"epoch": 0.38,
|
| 949 |
+
"grad_norm": 0.40234375,
|
| 950 |
+
"learning_rate": 3.879232514440227e-07,
|
| 951 |
+
"logits/chosen": 0.04711779206991196,
|
| 952 |
+
"logits/rejected": 0.9458179473876953,
|
| 953 |
+
"logps/chosen": -280.1850280761719,
|
| 954 |
+
"logps/rejected": -221.70785522460938,
|
| 955 |
+
"loss": 0.6533,
|
| 956 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 957 |
+
"rewards/chosen": 0.07137273252010345,
|
| 958 |
+
"rewards/margins": 0.0817800909280777,
|
| 959 |
+
"rewards/margins_max": 0.12524864077568054,
|
| 960 |
+
"rewards/margins_min": 0.038311541080474854,
|
| 961 |
+
"rewards/margins_std": 0.06147381663322449,
|
| 962 |
+
"rewards/rejected": -0.01040735375136137,
|
| 963 |
+
"step": 520
|
| 964 |
+
},
|
| 965 |
+
{
|
| 966 |
+
"epoch": 0.39,
|
| 967 |
+
"grad_norm": 0.51171875,
|
| 968 |
+
"learning_rate": 3.825221718033129e-07,
|
| 969 |
+
"logits/chosen": 0.09109713137149811,
|
| 970 |
+
"logits/rejected": 0.7228142023086548,
|
| 971 |
+
"logps/chosen": -256.89202880859375,
|
| 972 |
+
"logps/rejected": -236.75460815429688,
|
| 973 |
+
"loss": 0.6552,
|
| 974 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 975 |
+
"rewards/chosen": 0.06750547885894775,
|
| 976 |
+
"rewards/margins": 0.0867571011185646,
|
| 977 |
+
"rewards/margins_max": 0.12318144738674164,
|
| 978 |
+
"rewards/margins_min": 0.050332747399806976,
|
| 979 |
+
"rewards/margins_std": 0.05151181295514107,
|
| 980 |
+
"rewards/rejected": -0.019251624122262,
|
| 981 |
+
"step": 530
|
| 982 |
+
},
|
| 983 |
+
{
|
| 984 |
+
"epoch": 0.4,
|
| 985 |
+
"grad_norm": 0.375,
|
| 986 |
+
"learning_rate": 3.7703365189160746e-07,
|
| 987 |
+
"logits/chosen": -0.09472191333770752,
|
| 988 |
+
"logits/rejected": 0.6297842264175415,
|
| 989 |
+
"logps/chosen": -261.3214416503906,
|
| 990 |
+
"logps/rejected": -244.3570556640625,
|
| 991 |
+
"loss": 0.6562,
|
| 992 |
+
"rewards/accuracies": 0.925000011920929,
|
| 993 |
+
"rewards/chosen": 0.07096521556377411,
|
| 994 |
+
"rewards/margins": 0.07419043034315109,
|
| 995 |
+
"rewards/margins_max": 0.1066773533821106,
|
| 996 |
+
"rewards/margins_min": 0.041703514754772186,
|
| 997 |
+
"rewards/margins_std": 0.04594343155622482,
|
| 998 |
+
"rewards/rejected": -0.0032252087257802486,
|
| 999 |
+
"step": 540
|
| 1000 |
+
},
|
| 1001 |
+
{
|
| 1002 |
+
"epoch": 0.4,
|
| 1003 |
+
"grad_norm": 0.396484375,
|
| 1004 |
+
"learning_rate": 3.714613131230587e-07,
|
| 1005 |
+
"logits/chosen": -0.002119688782840967,
|
| 1006 |
+
"logits/rejected": 0.6269344091415405,
|
| 1007 |
+
"logps/chosen": -249.4778594970703,
|
| 1008 |
+
"logps/rejected": -223.02603149414062,
|
| 1009 |
+
"loss": 0.6572,
|
| 1010 |
+
"rewards/accuracies": 0.925000011920929,
|
| 1011 |
+
"rewards/chosen": 0.06649978458881378,
|
| 1012 |
+
"rewards/margins": 0.07572688907384872,
|
| 1013 |
+
"rewards/margins_max": 0.10936542600393295,
|
| 1014 |
+
"rewards/margins_min": 0.0420883484184742,
|
| 1015 |
+
"rewards/margins_std": 0.04757208377122879,
|
| 1016 |
+
"rewards/rejected": -0.009227104485034943,
|
| 1017 |
+
"step": 550
|
| 1018 |
+
},
|
| 1019 |
+
{
|
| 1020 |
+
"epoch": 0.41,
|
| 1021 |
+
"grad_norm": 0.427734375,
|
| 1022 |
+
"learning_rate": 3.6580883221685533e-07,
|
| 1023 |
+
"logits/chosen": 0.05928317829966545,
|
| 1024 |
+
"logits/rejected": 0.6542156338691711,
|
| 1025 |
+
"logps/chosen": -249.701171875,
|
| 1026 |
+
"logps/rejected": -235.28256225585938,
|
| 1027 |
+
"loss": 0.6546,
|
| 1028 |
+
"rewards/accuracies": 0.949999988079071,
|
| 1029 |
+
"rewards/chosen": 0.06236664578318596,
|
| 1030 |
+
"rewards/margins": 0.07323845475912094,
|
| 1031 |
+
"rewards/margins_max": 0.11050041764974594,
|
| 1032 |
+
"rewards/margins_min": 0.03597649186849594,
|
| 1033 |
+
"rewards/margins_std": 0.05269637703895569,
|
| 1034 |
+
"rewards/rejected": -0.010871809907257557,
|
| 1035 |
+
"step": 560
|
| 1036 |
+
},
|
| 1037 |
+
{
|
| 1038 |
+
"epoch": 0.42,
|
| 1039 |
+
"grad_norm": 0.39453125,
|
| 1040 |
+
"learning_rate": 3.6007993877126386e-07,
|
| 1041 |
+
"logits/chosen": -0.15580764412879944,
|
| 1042 |
+
"logits/rejected": 0.7158504724502563,
|
| 1043 |
+
"logps/chosen": -291.7669372558594,
|
| 1044 |
+
"logps/rejected": -232.95767211914062,
|
| 1045 |
+
"loss": 0.6561,
|
| 1046 |
+
"rewards/accuracies": 0.887499988079071,
|
| 1047 |
+
"rewards/chosen": 0.06417657434940338,
|
| 1048 |
+
"rewards/margins": 0.07455410063266754,
|
| 1049 |
+
"rewards/margins_max": 0.11633304506540298,
|
| 1050 |
+
"rewards/margins_min": 0.032775141298770905,
|
| 1051 |
+
"rewards/margins_std": 0.05908435583114624,
|
| 1052 |
+
"rewards/rejected": -0.010377511382102966,
|
| 1053 |
+
"step": 570
|
| 1054 |
+
},
|
| 1055 |
+
{
|
| 1056 |
+
"epoch": 0.43,
|
| 1057 |
+
"grad_norm": 0.490234375,
|
| 1058 |
+
"learning_rate": 3.5427841280277937e-07,
|
| 1059 |
+
"logits/chosen": 0.07802899181842804,
|
| 1060 |
+
"logits/rejected": 0.6800590753555298,
|
| 1061 |
+
"logps/chosen": -253.85107421875,
|
| 1062 |
+
"logps/rejected": -234.7869110107422,
|
| 1063 |
+
"loss": 0.6586,
|
| 1064 |
+
"rewards/accuracies": 0.9125000238418579,
|
| 1065 |
+
"rewards/chosen": 0.058535121381282806,
|
| 1066 |
+
"rewards/margins": 0.06843312084674835,
|
| 1067 |
+
"rewards/margins_max": 0.10552020370960236,
|
| 1068 |
+
"rewards/margins_min": 0.03134603053331375,
|
| 1069 |
+
"rewards/margins_std": 0.0524490587413311,
|
| 1070 |
+
"rewards/rejected": -0.0098979901522398,
|
| 1071 |
+
"step": 580
|
| 1072 |
+
},
|
| 1073 |
+
{
|
| 1074 |
+
"epoch": 0.43,
|
| 1075 |
+
"grad_norm": 0.482421875,
|
| 1076 |
+
"learning_rate": 3.484080822520096e-07,
|
| 1077 |
+
"logits/chosen": -0.049510225653648376,
|
| 1078 |
+
"logits/rejected": 0.7604032754898071,
|
| 1079 |
+
"logps/chosen": -268.8267822265625,
|
| 1080 |
+
"logps/rejected": -224.325439453125,
|
| 1081 |
+
"loss": 0.654,
|
| 1082 |
+
"rewards/accuracies": 0.9375,
|
| 1083 |
+
"rewards/chosen": 0.07341635227203369,
|
| 1084 |
+
"rewards/margins": 0.07777970284223557,
|
| 1085 |
+
"rewards/margins_max": 0.11911604553461075,
|
| 1086 |
+
"rewards/margins_min": 0.03644336014986038,
|
| 1087 |
+
"rewards/margins_std": 0.058458417654037476,
|
| 1088 |
+
"rewards/rejected": -0.004363355692476034,
|
| 1089 |
+
"step": 590
|
| 1090 |
+
},
|
| 1091 |
+
{
|
| 1092 |
+
"epoch": 0.44,
|
| 1093 |
+
"grad_norm": 0.400390625,
|
| 1094 |
+
"learning_rate": 3.4247282045793797e-07,
|
| 1095 |
+
"logits/chosen": -0.19325688481330872,
|
| 1096 |
+
"logits/rejected": 0.6270472407341003,
|
| 1097 |
+
"logps/chosen": -282.7298889160156,
|
| 1098 |
+
"logps/rejected": -252.6767578125,
|
| 1099 |
+
"loss": 0.6482,
|
| 1100 |
+
"rewards/accuracies": 0.987500011920929,
|
| 1101 |
+
"rewards/chosen": 0.07601656764745712,
|
| 1102 |
+
"rewards/margins": 0.09246112406253815,
|
| 1103 |
+
"rewards/margins_max": 0.13728852570056915,
|
| 1104 |
+
"rewards/margins_min": 0.04763372614979744,
|
| 1105 |
+
"rewards/margins_std": 0.06339551508426666,
|
| 1106 |
+
"rewards/rejected": -0.016444550827145576,
|
| 1107 |
+
"step": 600
|
| 1108 |
+
},
|
| 1109 |
+
{
|
| 1110 |
+
"epoch": 0.45,
|
| 1111 |
+
"grad_norm": 0.458984375,
|
| 1112 |
+
"learning_rate": 3.3647654360223144e-07,
|
| 1113 |
+
"logits/chosen": -0.05807774141430855,
|
| 1114 |
+
"logits/rejected": 0.6809174418449402,
|
| 1115 |
+
"logps/chosen": -299.19000244140625,
|
| 1116 |
+
"logps/rejected": -262.0146179199219,
|
| 1117 |
+
"loss": 0.6554,
|
| 1118 |
+
"rewards/accuracies": 0.9375,
|
| 1119 |
+
"rewards/chosen": 0.06890231370925903,
|
| 1120 |
+
"rewards/margins": 0.07707327604293823,
|
| 1121 |
+
"rewards/margins_max": 0.11320241540670395,
|
| 1122 |
+
"rewards/margins_min": 0.04094412922859192,
|
| 1123 |
+
"rewards/margins_std": 0.05109432339668274,
|
| 1124 |
+
"rewards/rejected": -0.0081709586083889,
|
| 1125 |
+
"step": 610
|
| 1126 |
+
},
|
| 1127 |
+
{
|
| 1128 |
+
"epoch": 0.46,
|
| 1129 |
+
"grad_norm": 0.4609375,
|
| 1130 |
+
"learning_rate": 3.30423208125281e-07,
|
| 1131 |
+
"logits/chosen": -0.15423426032066345,
|
| 1132 |
+
"logits/rejected": 0.6746962070465088,
|
| 1133 |
+
"logps/chosen": -361.3031921386719,
|
| 1134 |
+
"logps/rejected": -271.8617858886719,
|
| 1135 |
+
"loss": 0.6527,
|
| 1136 |
+
"rewards/accuracies": 0.987500011920929,
|
| 1137 |
+
"rewards/chosen": 0.08182433992624283,
|
| 1138 |
+
"rewards/margins": 0.08147110044956207,
|
| 1139 |
+
"rewards/margins_max": 0.1197957769036293,
|
| 1140 |
+
"rewards/margins_min": 0.04314642399549484,
|
| 1141 |
+
"rewards/margins_std": 0.05419927090406418,
|
| 1142 |
+
"rewards/rejected": 0.00035324436612427235,
|
| 1143 |
+
"step": 620
|
| 1144 |
+
},
|
| 1145 |
+
{
|
| 1146 |
+
"epoch": 0.46,
|
| 1147 |
+
"grad_norm": 0.53515625,
|
| 1148 |
+
"learning_rate": 3.2431680811567833e-07,
|
| 1149 |
+
"logits/chosen": -0.18638429045677185,
|
| 1150 |
+
"logits/rejected": 0.5596809387207031,
|
| 1151 |
+
"logps/chosen": -265.38189697265625,
|
| 1152 |
+
"logps/rejected": -216.4676513671875,
|
| 1153 |
+
"loss": 0.6498,
|
| 1154 |
+
"rewards/accuracies": 0.9375,
|
| 1155 |
+
"rewards/chosen": 0.07413068413734436,
|
| 1156 |
+
"rewards/margins": 0.08364250510931015,
|
| 1157 |
+
"rewards/margins_max": 0.11661893129348755,
|
| 1158 |
+
"rewards/margins_min": 0.05066608637571335,
|
| 1159 |
+
"rewards/margins_std": 0.046635713428258896,
|
| 1160 |
+
"rewards/rejected": -0.009511834010481834,
|
| 1161 |
+
"step": 630
|
| 1162 |
+
},
|
| 1163 |
+
{
|
| 1164 |
+
"epoch": 0.47,
|
| 1165 |
+
"grad_norm": 0.4765625,
|
| 1166 |
+
"learning_rate": 3.1816137267485136e-07,
|
| 1167 |
+
"logits/chosen": -0.01266663521528244,
|
| 1168 |
+
"logits/rejected": 0.7700913548469543,
|
| 1169 |
+
"logps/chosen": -270.8904113769531,
|
| 1170 |
+
"logps/rejected": -227.817138671875,
|
| 1171 |
+
"loss": 0.6499,
|
| 1172 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 1173 |
+
"rewards/chosen": 0.08034569025039673,
|
| 1174 |
+
"rewards/margins": 0.09530878067016602,
|
| 1175 |
+
"rewards/margins_max": 0.13598743081092834,
|
| 1176 |
+
"rewards/margins_min": 0.05463013797998428,
|
| 1177 |
+
"rewards/margins_std": 0.0575282946228981,
|
| 1178 |
+
"rewards/rejected": -0.014963088557124138,
|
| 1179 |
+
"step": 640
|
| 1180 |
+
},
|
| 1181 |
+
{
|
| 1182 |
+
"epoch": 0.48,
|
| 1183 |
+
"grad_norm": 0.458984375,
|
| 1184 |
+
"learning_rate": 3.1196096325859815e-07,
|
| 1185 |
+
"logits/chosen": -0.09016970545053482,
|
| 1186 |
+
"logits/rejected": 0.5758002400398254,
|
| 1187 |
+
"logps/chosen": -248.17721557617188,
|
| 1188 |
+
"logps/rejected": -231.0363311767578,
|
| 1189 |
+
"loss": 0.6539,
|
| 1190 |
+
"rewards/accuracies": 0.925000011920929,
|
| 1191 |
+
"rewards/chosen": 0.06288131326436996,
|
| 1192 |
+
"rewards/margins": 0.08216816931962967,
|
| 1193 |
+
"rewards/margins_max": 0.12130717188119888,
|
| 1194 |
+
"rewards/margins_min": 0.04302916303277016,
|
| 1195 |
+
"rewards/margins_std": 0.05535091087222099,
|
| 1196 |
+
"rewards/rejected": -0.019286850467324257,
|
| 1197 |
+
"step": 650
|
| 1198 |
+
},
|
| 1199 |
+
{
|
| 1200 |
+
"epoch": 0.49,
|
| 1201 |
+
"grad_norm": 0.357421875,
|
| 1202 |
+
"learning_rate": 3.057196709972727e-07,
|
| 1203 |
+
"logits/chosen": -0.06700615584850311,
|
| 1204 |
+
"logits/rejected": 0.7238128781318665,
|
| 1205 |
+
"logps/chosen": -273.21044921875,
|
| 1206 |
+
"logps/rejected": -222.5702667236328,
|
| 1207 |
+
"loss": 0.6541,
|
| 1208 |
+
"rewards/accuracies": 0.925000011920929,
|
| 1209 |
+
"rewards/chosen": 0.07693549245595932,
|
| 1210 |
+
"rewards/margins": 0.08857451379299164,
|
| 1211 |
+
"rewards/margins_max": 0.13141193985939026,
|
| 1212 |
+
"rewards/margins_min": 0.045737091451883316,
|
| 1213 |
+
"rewards/margins_std": 0.0605812594294548,
|
| 1214 |
+
"rewards/rejected": -0.011639016680419445,
|
| 1215 |
+
"step": 660
|
| 1216 |
+
},
|
| 1217 |
+
{
|
| 1218 |
+
"epoch": 0.49,
|
| 1219 |
+
"grad_norm": 0.384765625,
|
| 1220 |
+
"learning_rate": 2.9944161399639086e-07,
|
| 1221 |
+
"logits/chosen": -0.17069509625434875,
|
| 1222 |
+
"logits/rejected": 0.5526743531227112,
|
| 1223 |
+
"logps/chosen": -274.2879943847656,
|
| 1224 |
+
"logps/rejected": -238.9512176513672,
|
| 1225 |
+
"loss": 0.6554,
|
| 1226 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 1227 |
+
"rewards/chosen": 0.07316674292087555,
|
| 1228 |
+
"rewards/margins": 0.0783548504114151,
|
| 1229 |
+
"rewards/margins_max": 0.12053277343511581,
|
| 1230 |
+
"rewards/margins_min": 0.03617694973945618,
|
| 1231 |
+
"rewards/margins_std": 0.059648580849170685,
|
| 1232 |
+
"rewards/rejected": -0.0051881177350878716,
|
| 1233 |
+
"step": 670
|
| 1234 |
+
},
|
| 1235 |
+
{
|
| 1236 |
+
"epoch": 0.5,
|
| 1237 |
+
"grad_norm": 0.4453125,
|
| 1238 |
+
"learning_rate": 2.9313093461943824e-07,
|
| 1239 |
+
"logits/chosen": -0.16692019999027252,
|
| 1240 |
+
"logits/rejected": 0.6132463216781616,
|
| 1241 |
+
"logps/chosen": -266.98297119140625,
|
| 1242 |
+
"logps/rejected": -240.8970489501953,
|
| 1243 |
+
"loss": 0.6514,
|
| 1244 |
+
"rewards/accuracies": 0.949999988079071,
|
| 1245 |
+
"rewards/chosen": 0.0738702192902565,
|
| 1246 |
+
"rewards/margins": 0.08596795052289963,
|
| 1247 |
+
"rewards/margins_max": 0.12484346330165863,
|
| 1248 |
+
"rewards/margins_min": 0.04709243029356003,
|
| 1249 |
+
"rewards/margins_std": 0.05497828125953674,
|
| 1250 |
+
"rewards/rejected": -0.012097733095288277,
|
| 1251 |
+
"step": 680
|
| 1252 |
+
},
|
| 1253 |
+
{
|
| 1254 |
+
"epoch": 0.51,
|
| 1255 |
+
"grad_norm": 0.404296875,
|
| 1256 |
+
"learning_rate": 2.8679179675467104e-07,
|
| 1257 |
+
"logits/chosen": -0.028652098029851913,
|
| 1258 |
+
"logits/rejected": 0.7112724781036377,
|
| 1259 |
+
"logps/chosen": -263.5270080566406,
|
| 1260 |
+
"logps/rejected": -232.5438232421875,
|
| 1261 |
+
"loss": 0.6505,
|
| 1262 |
+
"rewards/accuracies": 0.9375,
|
| 1263 |
+
"rewards/chosen": 0.07920118421316147,
|
| 1264 |
+
"rewards/margins": 0.08954565227031708,
|
| 1265 |
+
"rewards/margins_max": 0.13445377349853516,
|
| 1266 |
+
"rewards/margins_min": 0.04463755339384079,
|
| 1267 |
+
"rewards/margins_std": 0.06350965052843094,
|
| 1268 |
+
"rewards/rejected": -0.010344477370381355,
|
| 1269 |
+
"step": 690
|
| 1270 |
+
},
|
| 1271 |
+
{
|
| 1272 |
+
"epoch": 0.52,
|
| 1273 |
+
"grad_norm": 0.44140625,
|
| 1274 |
+
"learning_rate": 2.80428383067716e-07,
|
| 1275 |
+
"logits/chosen": 0.04036666825413704,
|
| 1276 |
+
"logits/rejected": 0.5802733302116394,
|
| 1277 |
+
"logps/chosen": -276.1773681640625,
|
| 1278 |
+
"logps/rejected": -284.2413024902344,
|
| 1279 |
+
"loss": 0.6492,
|
| 1280 |
+
"rewards/accuracies": 0.949999988079071,
|
| 1281 |
+
"rewards/chosen": 0.07053406536579132,
|
| 1282 |
+
"rewards/margins": 0.09340489655733109,
|
| 1283 |
+
"rewards/margins_max": 0.13880935311317444,
|
| 1284 |
+
"rewards/margins_min": 0.048000432550907135,
|
| 1285 |
+
"rewards/margins_std": 0.06421160697937012,
|
| 1286 |
+
"rewards/rejected": -0.022870829328894615,
|
| 1287 |
+
"step": 700
|
| 1288 |
+
},
|
| 1289 |
+
{
|
| 1290 |
+
"epoch": 0.52,
|
| 1291 |
+
"grad_norm": 0.484375,
|
| 1292 |
+
"learning_rate": 2.7404489224177973e-07,
|
| 1293 |
+
"logits/chosen": -0.10626095533370972,
|
| 1294 |
+
"logits/rejected": 0.6656783819198608,
|
| 1295 |
+
"logps/chosen": -273.10369873046875,
|
| 1296 |
+
"logps/rejected": -240.2863006591797,
|
| 1297 |
+
"loss": 0.6555,
|
| 1298 |
+
"rewards/accuracies": 0.925000011920929,
|
| 1299 |
+
"rewards/chosen": 0.06227077171206474,
|
| 1300 |
+
"rewards/margins": 0.07790277898311615,
|
| 1301 |
+
"rewards/margins_max": 0.12013592571020126,
|
| 1302 |
+
"rewards/margins_min": 0.03566960617899895,
|
| 1303 |
+
"rewards/margins_std": 0.05972672253847122,
|
| 1304 |
+
"rewards/rejected": -0.015631992369890213,
|
| 1305 |
+
"step": 710
|
| 1306 |
+
},
|
| 1307 |
+
{
|
| 1308 |
+
"epoch": 0.53,
|
| 1309 |
+
"grad_norm": 0.392578125,
|
| 1310 |
+
"learning_rate": 2.676455362072894e-07,
|
| 1311 |
+
"logits/chosen": -0.09553918987512589,
|
| 1312 |
+
"logits/rejected": 0.7383521795272827,
|
| 1313 |
+
"logps/chosen": -302.948974609375,
|
| 1314 |
+
"logps/rejected": -269.911376953125,
|
| 1315 |
+
"loss": 0.6506,
|
| 1316 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 1317 |
+
"rewards/chosen": 0.08486850559711456,
|
| 1318 |
+
"rewards/margins": 0.10140830278396606,
|
| 1319 |
+
"rewards/margins_max": 0.14273163676261902,
|
| 1320 |
+
"rewards/margins_min": 0.06008495017886162,
|
| 1321 |
+
"rewards/margins_std": 0.05844002962112427,
|
| 1322 |
+
"rewards/rejected": -0.016539786010980606,
|
| 1323 |
+
"step": 720
|
| 1324 |
+
},
|
| 1325 |
+
{
|
| 1326 |
+
"epoch": 0.54,
|
| 1327 |
+
"grad_norm": 0.37890625,
|
| 1328 |
+
"learning_rate": 2.612345373627937e-07,
|
| 1329 |
+
"logits/chosen": 0.11825668811798096,
|
| 1330 |
+
"logits/rejected": 0.7459693551063538,
|
| 1331 |
+
"logps/chosen": -275.9749450683594,
|
| 1332 |
+
"logps/rejected": -238.9975128173828,
|
| 1333 |
+
"loss": 0.6551,
|
| 1334 |
+
"rewards/accuracies": 0.9375,
|
| 1335 |
+
"rewards/chosen": 0.06423775851726532,
|
| 1336 |
+
"rewards/margins": 0.08155209571123123,
|
| 1337 |
+
"rewards/margins_max": 0.12171584367752075,
|
| 1338 |
+
"rewards/margins_min": 0.04138834401965141,
|
| 1339 |
+
"rewards/margins_std": 0.05680011585354805,
|
| 1340 |
+
"rewards/rejected": -0.017314324155449867,
|
| 1341 |
+
"step": 730
|
| 1342 |
+
},
|
| 1343 |
+
{
|
| 1344 |
+
"epoch": 0.54,
|
| 1345 |
+
"grad_norm": 0.40234375,
|
| 1346 |
+
"learning_rate": 2.54816125788955e-07,
|
| 1347 |
+
"logits/chosen": -0.04539821669459343,
|
| 1348 |
+
"logits/rejected": 0.8194332122802734,
|
| 1349 |
+
"logps/chosen": -290.15557861328125,
|
| 1350 |
+
"logps/rejected": -230.5435333251953,
|
| 1351 |
+
"loss": 0.6515,
|
| 1352 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 1353 |
+
"rewards/chosen": 0.074457548558712,
|
| 1354 |
+
"rewards/margins": 0.08972884714603424,
|
| 1355 |
+
"rewards/margins_max": 0.13167297840118408,
|
| 1356 |
+
"rewards/margins_min": 0.04778473079204559,
|
| 1357 |
+
"rewards/margins_std": 0.059317946434020996,
|
| 1358 |
+
"rewards/rejected": -0.015271296724677086,
|
| 1359 |
+
"step": 740
|
| 1360 |
+
},
|
| 1361 |
+
{
|
| 1362 |
+
"epoch": 0.55,
|
| 1363 |
+
"grad_norm": 0.38671875,
|
| 1364 |
+
"learning_rate": 2.4839453645747467e-07,
|
| 1365 |
+
"logits/chosen": 0.04359906539320946,
|
| 1366 |
+
"logits/rejected": 0.622580349445343,
|
| 1367 |
+
"logps/chosen": -276.761474609375,
|
| 1368 |
+
"logps/rejected": -252.15261840820312,
|
| 1369 |
+
"loss": 0.6543,
|
| 1370 |
+
"rewards/accuracies": 0.987500011920929,
|
| 1371 |
+
"rewards/chosen": 0.07819774001836777,
|
| 1372 |
+
"rewards/margins": 0.0873616486787796,
|
| 1373 |
+
"rewards/margins_max": 0.14198613166809082,
|
| 1374 |
+
"rewards/margins_min": 0.03273715078830719,
|
| 1375 |
+
"rewards/margins_std": 0.07725068926811218,
|
| 1376 |
+
"rewards/rejected": -0.009163893759250641,
|
| 1377 |
+
"step": 750
|
| 1378 |
+
},
|
| 1379 |
+
{
|
| 1380 |
+
"epoch": 0.56,
|
| 1381 |
+
"grad_norm": 0.33984375,
|
| 1382 |
+
"learning_rate": 2.4197400643678987e-07,
|
| 1383 |
+
"logits/chosen": 0.03581676632165909,
|
| 1384 |
+
"logits/rejected": 0.8077651858329773,
|
| 1385 |
+
"logps/chosen": -280.2861328125,
|
| 1386 |
+
"logps/rejected": -233.9840545654297,
|
| 1387 |
+
"loss": 0.6528,
|
| 1388 |
+
"rewards/accuracies": 0.987500011920929,
|
| 1389 |
+
"rewards/chosen": 0.07635350525379181,
|
| 1390 |
+
"rewards/margins": 0.09026426076889038,
|
| 1391 |
+
"rewards/margins_max": 0.1272718459367752,
|
| 1392 |
+
"rewards/margins_min": 0.05325668305158615,
|
| 1393 |
+
"rewards/margins_std": 0.05233662202954292,
|
| 1394 |
+
"rewards/rejected": -0.013910764828324318,
|
| 1395 |
+
"step": 760
|
| 1396 |
+
},
|
| 1397 |
+
{
|
| 1398 |
+
"epoch": 0.57,
|
| 1399 |
+
"grad_norm": 0.435546875,
|
| 1400 |
+
"learning_rate": 2.3555877209638726e-07,
|
| 1401 |
+
"logits/chosen": -0.12947851419448853,
|
| 1402 |
+
"logits/rejected": 0.7489079833030701,
|
| 1403 |
+
"logps/chosen": -290.2663879394531,
|
| 1404 |
+
"logps/rejected": -254.322021484375,
|
| 1405 |
+
"loss": 0.6506,
|
| 1406 |
+
"rewards/accuracies": 0.9375,
|
| 1407 |
+
"rewards/chosen": 0.0825168713927269,
|
| 1408 |
+
"rewards/margins": 0.08864180743694305,
|
| 1409 |
+
"rewards/margins_max": 0.12674041092395782,
|
| 1410 |
+
"rewards/margins_min": 0.050543200224637985,
|
| 1411 |
+
"rewards/margins_std": 0.05387955904006958,
|
| 1412 |
+
"rewards/rejected": -0.006124935112893581,
|
| 1413 |
+
"step": 770
|
| 1414 |
+
},
|
| 1415 |
+
{
|
| 1416 |
+
"epoch": 0.57,
|
| 1417 |
+
"grad_norm": 0.390625,
|
| 1418 |
+
"learning_rate": 2.2915306631157817e-07,
|
| 1419 |
+
"logits/chosen": -0.04638870805501938,
|
| 1420 |
+
"logits/rejected": 0.6960999965667725,
|
| 1421 |
+
"logps/chosen": -238.04745483398438,
|
| 1422 |
+
"logps/rejected": -215.9326629638672,
|
| 1423 |
+
"loss": 0.6497,
|
| 1424 |
+
"rewards/accuracies": 0.949999988079071,
|
| 1425 |
+
"rewards/chosen": 0.06742905080318451,
|
| 1426 |
+
"rewards/margins": 0.08723022788763046,
|
| 1427 |
+
"rewards/margins_max": 0.13323740661144257,
|
| 1428 |
+
"rewards/margins_min": 0.04122306779026985,
|
| 1429 |
+
"rewards/margins_std": 0.0650639608502388,
|
| 1430 |
+
"rewards/rejected": -0.019801190122961998,
|
| 1431 |
+
"step": 780
|
| 1432 |
+
},
|
| 1433 |
+
{
|
| 1434 |
+
"epoch": 0.58,
|
| 1435 |
+
"grad_norm": 0.451171875,
|
| 1436 |
+
"learning_rate": 2.2276111567057887e-07,
|
| 1437 |
+
"logits/chosen": 0.19736722111701965,
|
| 1438 |
+
"logits/rejected": 0.8112510442733765,
|
| 1439 |
+
"logps/chosen": -263.0642395019531,
|
| 1440 |
+
"logps/rejected": -247.5417022705078,
|
| 1441 |
+
"loss": 0.6481,
|
| 1442 |
+
"rewards/accuracies": 0.9375,
|
| 1443 |
+
"rewards/chosen": 0.07030580937862396,
|
| 1444 |
+
"rewards/margins": 0.08302759379148483,
|
| 1445 |
+
"rewards/margins_max": 0.12399895489215851,
|
| 1446 |
+
"rewards/margins_min": 0.042056240141391754,
|
| 1447 |
+
"rewards/margins_std": 0.05794224888086319,
|
| 1448 |
+
"rewards/rejected": -0.01272178627550602,
|
| 1449 |
+
"step": 790
|
| 1450 |
+
},
|
| 1451 |
+
{
|
| 1452 |
+
"epoch": 0.59,
|
| 1453 |
+
"grad_norm": 0.474609375,
|
| 1454 |
+
"learning_rate": 2.1638713768573936e-07,
|
| 1455 |
+
"logits/chosen": 0.0460067018866539,
|
| 1456 |
+
"logits/rejected": 0.6669069528579712,
|
| 1457 |
+
"logps/chosen": -227.3536834716797,
|
| 1458 |
+
"logps/rejected": -220.83984375,
|
| 1459 |
+
"loss": 0.6476,
|
| 1460 |
+
"rewards/accuracies": 0.925000011920929,
|
| 1461 |
+
"rewards/chosen": 0.0669163167476654,
|
| 1462 |
+
"rewards/margins": 0.07873805612325668,
|
| 1463 |
+
"rewards/margins_max": 0.12541484832763672,
|
| 1464 |
+
"rewards/margins_min": 0.03206125646829605,
|
| 1465 |
+
"rewards/margins_std": 0.0660109594464302,
|
| 1466 |
+
"rewards/rejected": -0.011821739375591278,
|
| 1467 |
+
"step": 800
|
| 1468 |
+
},
|
| 1469 |
+
{
|
| 1470 |
+
"epoch": 0.6,
|
| 1471 |
+
"grad_norm": 0.484375,
|
| 1472 |
+
"learning_rate": 2.100353380107609e-07,
|
| 1473 |
+
"logits/chosen": -0.007617829833179712,
|
| 1474 |
+
"logits/rejected": 0.7070167660713196,
|
| 1475 |
+
"logps/chosen": -271.02691650390625,
|
| 1476 |
+
"logps/rejected": -219.64944458007812,
|
| 1477 |
+
"loss": 0.655,
|
| 1478 |
+
"rewards/accuracies": 0.925000011920929,
|
| 1479 |
+
"rewards/chosen": 0.05975126102566719,
|
| 1480 |
+
"rewards/margins": 0.07805275171995163,
|
| 1481 |
+
"rewards/margins_max": 0.11311063915491104,
|
| 1482 |
+
"rewards/margins_min": 0.04299486428499222,
|
| 1483 |
+
"rewards/margins_std": 0.04957933351397514,
|
| 1484 |
+
"rewards/rejected": -0.01830149069428444,
|
| 1485 |
+
"step": 810
|
| 1486 |
+
},
|
| 1487 |
+
{
|
| 1488 |
+
"epoch": 0.6,
|
| 1489 |
+
"grad_norm": 0.400390625,
|
| 1490 |
+
"learning_rate": 2.0370990766573698e-07,
|
| 1491 |
+
"logits/chosen": 0.11678284406661987,
|
| 1492 |
+
"logits/rejected": 0.8694367408752441,
|
| 1493 |
+
"logps/chosen": -319.4522705078125,
|
| 1494 |
+
"logps/rejected": -231.9202423095703,
|
| 1495 |
+
"loss": 0.653,
|
| 1496 |
+
"rewards/accuracies": 0.925000011920929,
|
| 1497 |
+
"rewards/chosen": 0.060270898044109344,
|
| 1498 |
+
"rewards/margins": 0.08039253205060959,
|
| 1499 |
+
"rewards/margins_max": 0.11875177919864655,
|
| 1500 |
+
"rewards/margins_min": 0.042033277451992035,
|
| 1501 |
+
"rewards/margins_std": 0.054248176515102386,
|
| 1502 |
+
"rewards/rejected": -0.020121637731790543,
|
| 1503 |
+
"step": 820
|
| 1504 |
+
},
|
| 1505 |
+
{
|
| 1506 |
+
"epoch": 0.61,
|
| 1507 |
+
"grad_norm": 0.357421875,
|
| 1508 |
+
"learning_rate": 1.974150202718513e-07,
|
| 1509 |
+
"logits/chosen": 0.037704307585954666,
|
| 1510 |
+
"logits/rejected": 0.6765593886375427,
|
| 1511 |
+
"logps/chosen": -252.72128295898438,
|
| 1512 |
+
"logps/rejected": -251.72232055664062,
|
| 1513 |
+
"loss": 0.6551,
|
| 1514 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 1515 |
+
"rewards/chosen": 0.06962188333272934,
|
| 1516 |
+
"rewards/margins": 0.07926660031080246,
|
| 1517 |
+
"rewards/margins_max": 0.11337349563837051,
|
| 1518 |
+
"rewards/margins_min": 0.0451597236096859,
|
| 1519 |
+
"rewards/margins_std": 0.04823441430926323,
|
| 1520 |
+
"rewards/rejected": -0.009644721634685993,
|
| 1521 |
+
"step": 830
|
| 1522 |
+
},
|
| 1523 |
+
{
|
| 1524 |
+
"epoch": 0.62,
|
| 1525 |
+
"grad_norm": 0.4296875,
|
| 1526 |
+
"learning_rate": 1.9115482929755445e-07,
|
| 1527 |
+
"logits/chosen": 0.21195778250694275,
|
| 1528 |
+
"logits/rejected": 0.8628422021865845,
|
| 1529 |
+
"logps/chosen": -255.17141723632812,
|
| 1530 |
+
"logps/rejected": -230.54837036132812,
|
| 1531 |
+
"loss": 0.6542,
|
| 1532 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 1533 |
+
"rewards/chosen": 0.05505312606692314,
|
| 1534 |
+
"rewards/margins": 0.08028857409954071,
|
| 1535 |
+
"rewards/margins_max": 0.12079598009586334,
|
| 1536 |
+
"rewards/margins_min": 0.03978116437792778,
|
| 1537 |
+
"rewards/margins_std": 0.05728612467646599,
|
| 1538 |
+
"rewards/rejected": -0.025235449895262718,
|
| 1539 |
+
"step": 840
|
| 1540 |
+
},
|
| 1541 |
+
{
|
| 1542 |
+
"epoch": 0.63,
|
| 1543 |
+
"grad_norm": 0.451171875,
|
| 1544 |
+
"learning_rate": 1.8493346531803887e-07,
|
| 1545 |
+
"logits/chosen": -0.0924936980009079,
|
| 1546 |
+
"logits/rejected": 0.6660225987434387,
|
| 1547 |
+
"logps/chosen": -253.78762817382812,
|
| 1548 |
+
"logps/rejected": -230.6340789794922,
|
| 1549 |
+
"loss": 0.654,
|
| 1550 |
+
"rewards/accuracies": 0.925000011920929,
|
| 1551 |
+
"rewards/chosen": 0.07504816353321075,
|
| 1552 |
+
"rewards/margins": 0.08388366550207138,
|
| 1553 |
+
"rewards/margins_max": 0.13147205114364624,
|
| 1554 |
+
"rewards/margins_min": 0.03629531338810921,
|
| 1555 |
+
"rewards/margins_std": 0.06730011850595474,
|
| 1556 |
+
"rewards/rejected": -0.00883551873266697,
|
| 1557 |
+
"step": 850
|
| 1558 |
+
},
|
| 1559 |
+
{
|
| 1560 |
+
"epoch": 0.63,
|
| 1561 |
+
"grad_norm": 0.62890625,
|
| 1562 |
+
"learning_rate": 1.7875503328981807e-07,
|
| 1563 |
+
"logits/chosen": 0.03313580900430679,
|
| 1564 |
+
"logits/rejected": 0.7846869826316833,
|
| 1565 |
+
"logps/chosen": -289.10919189453125,
|
| 1566 |
+
"logps/rejected": -260.46209716796875,
|
| 1567 |
+
"loss": 0.651,
|
| 1568 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 1569 |
+
"rewards/chosen": 0.07691006362438202,
|
| 1570 |
+
"rewards/margins": 0.09373887628316879,
|
| 1571 |
+
"rewards/margins_max": 0.14286434650421143,
|
| 1572 |
+
"rewards/margins_min": 0.04461339861154556,
|
| 1573 |
+
"rewards/margins_std": 0.06947391480207443,
|
| 1574 |
+
"rewards/rejected": -0.01682879962027073,
|
| 1575 |
+
"step": 860
|
| 1576 |
+
},
|
| 1577 |
+
{
|
| 1578 |
+
"epoch": 0.64,
|
| 1579 |
+
"grad_norm": 0.4765625,
|
| 1580 |
+
"learning_rate": 1.7262360984221006e-07,
|
| 1581 |
+
"logits/chosen": 0.07170330733060837,
|
| 1582 |
+
"logits/rejected": 0.585077166557312,
|
| 1583 |
+
"logps/chosen": -232.71224975585938,
|
| 1584 |
+
"logps/rejected": -241.5399932861328,
|
| 1585 |
+
"loss": 0.649,
|
| 1586 |
+
"rewards/accuracies": 0.949999988079071,
|
| 1587 |
+
"rewards/chosen": 0.06494490802288055,
|
| 1588 |
+
"rewards/margins": 0.082070492208004,
|
| 1589 |
+
"rewards/margins_max": 0.11704879999160767,
|
| 1590 |
+
"rewards/margins_min": 0.04709218442440033,
|
| 1591 |
+
"rewards/margins_std": 0.049466803669929504,
|
| 1592 |
+
"rewards/rejected": -0.017125584185123444,
|
| 1593 |
+
"step": 870
|
| 1594 |
+
},
|
| 1595 |
+
{
|
| 1596 |
+
"epoch": 0.65,
|
| 1597 |
+
"grad_norm": 0.46875,
|
| 1598 |
+
"learning_rate": 1.6654324058751175e-07,
|
| 1599 |
+
"logits/chosen": -0.03041740693151951,
|
| 1600 |
+
"logits/rejected": 0.655995786190033,
|
| 1601 |
+
"logps/chosen": -307.1095275878906,
|
| 1602 |
+
"logps/rejected": -223.6023712158203,
|
| 1603 |
+
"loss": 0.6511,
|
| 1604 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 1605 |
+
"rewards/chosen": 0.05577283352613449,
|
| 1606 |
+
"rewards/margins": 0.0676901713013649,
|
| 1607 |
+
"rewards/margins_max": 0.10265400260686874,
|
| 1608 |
+
"rewards/margins_min": 0.03272632881999016,
|
| 1609 |
+
"rewards/margins_std": 0.04944632947444916,
|
| 1610 |
+
"rewards/rejected": -0.011917328462004662,
|
| 1611 |
+
"step": 880
|
| 1612 |
+
},
|
| 1613 |
+
{
|
| 1614 |
+
"epoch": 0.65,
|
| 1615 |
+
"grad_norm": 0.439453125,
|
| 1616 |
+
"learning_rate": 1.6051793745163812e-07,
|
| 1617 |
+
"logits/chosen": -0.03592555597424507,
|
| 1618 |
+
"logits/rejected": 0.5853763222694397,
|
| 1619 |
+
"logps/chosen": -272.7641296386719,
|
| 1620 |
+
"logps/rejected": -243.679443359375,
|
| 1621 |
+
"loss": 0.6543,
|
| 1622 |
+
"rewards/accuracies": 0.949999988079071,
|
| 1623 |
+
"rewards/chosen": 0.07209287583827972,
|
| 1624 |
+
"rewards/margins": 0.0827474445104599,
|
| 1625 |
+
"rewards/margins_max": 0.11978437751531601,
|
| 1626 |
+
"rewards/margins_min": 0.045710500329732895,
|
| 1627 |
+
"rewards/margins_std": 0.05237815901637077,
|
| 1628 |
+
"rewards/rejected": -0.010654577054083347,
|
| 1629 |
+
"step": 890
|
| 1630 |
+
},
|
| 1631 |
+
{
|
| 1632 |
+
"epoch": 0.66,
|
| 1633 |
+
"grad_norm": 0.435546875,
|
| 1634 |
+
"learning_rate": 1.5455167602698915e-07,
|
| 1635 |
+
"logits/chosen": 0.10257561504840851,
|
| 1636 |
+
"logits/rejected": 0.7078714370727539,
|
| 1637 |
+
"logps/chosen": -239.56201171875,
|
| 1638 |
+
"logps/rejected": -240.7050323486328,
|
| 1639 |
+
"loss": 0.6508,
|
| 1640 |
+
"rewards/accuracies": 0.949999988079071,
|
| 1641 |
+
"rewards/chosen": 0.06524372100830078,
|
| 1642 |
+
"rewards/margins": 0.08618183434009552,
|
| 1643 |
+
"rewards/margins_max": 0.12894438207149506,
|
| 1644 |
+
"rewards/margins_min": 0.043419282883405685,
|
| 1645 |
+
"rewards/margins_std": 0.060475386679172516,
|
| 1646 |
+
"rewards/rejected": -0.020938115194439888,
|
| 1647 |
+
"step": 900
|
| 1648 |
+
},
|
| 1649 |
+
{
|
| 1650 |
+
"epoch": 0.67,
|
| 1651 |
+
"grad_norm": 0.5,
|
| 1652 |
+
"learning_rate": 1.4864839294928924e-07,
|
| 1653 |
+
"logits/chosen": 0.03240091726183891,
|
| 1654 |
+
"logits/rejected": 0.7821232080459595,
|
| 1655 |
+
"logps/chosen": -283.28338623046875,
|
| 1656 |
+
"logps/rejected": -228.65771484375,
|
| 1657 |
+
"loss": 0.6537,
|
| 1658 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 1659 |
+
"rewards/chosen": 0.06496749818325043,
|
| 1660 |
+
"rewards/margins": 0.08208582550287247,
|
| 1661 |
+
"rewards/margins_max": 0.12210579216480255,
|
| 1662 |
+
"rewards/margins_min": 0.04206586629152298,
|
| 1663 |
+
"rewards/margins_std": 0.056596774607896805,
|
| 1664 |
+
"rewards/rejected": -0.01711832545697689,
|
| 1665 |
+
"step": 910
|
| 1666 |
+
},
|
| 1667 |
+
{
|
| 1668 |
+
"epoch": 0.68,
|
| 1669 |
+
"grad_norm": 0.515625,
|
| 1670 |
+
"learning_rate": 1.428119833001315e-07,
|
| 1671 |
+
"logits/chosen": -0.23715606331825256,
|
| 1672 |
+
"logits/rejected": 0.6829497218132019,
|
| 1673 |
+
"logps/chosen": -352.04071044921875,
|
| 1674 |
+
"logps/rejected": -288.7782287597656,
|
| 1675 |
+
"loss": 0.6512,
|
| 1676 |
+
"rewards/accuracies": 0.987500011920929,
|
| 1677 |
+
"rewards/chosen": 0.07856696844100952,
|
| 1678 |
+
"rewards/margins": 0.08786121755838394,
|
| 1679 |
+
"rewards/margins_max": 0.1261076182126999,
|
| 1680 |
+
"rewards/margins_min": 0.049614839255809784,
|
| 1681 |
+
"rewards/margins_std": 0.05408855527639389,
|
| 1682 |
+
"rewards/rejected": -0.009294259361922741,
|
| 1683 |
+
"step": 920
|
| 1684 |
+
},
|
| 1685 |
+
{
|
| 1686 |
+
"epoch": 0.68,
|
| 1687 |
+
"grad_norm": 0.41796875,
|
| 1688 |
+
"learning_rate": 1.370462980369401e-07,
|
| 1689 |
+
"logits/chosen": 0.029560232535004616,
|
| 1690 |
+
"logits/rejected": 0.729602038860321,
|
| 1691 |
+
"logps/chosen": -257.4678955078125,
|
| 1692 |
+
"logps/rejected": -246.0548858642578,
|
| 1693 |
+
"loss": 0.6487,
|
| 1694 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 1695 |
+
"rewards/chosen": 0.0831000953912735,
|
| 1696 |
+
"rewards/margins": 0.09744415432214737,
|
| 1697 |
+
"rewards/margins_max": 0.15836606919765472,
|
| 1698 |
+
"rewards/margins_min": 0.03652223199605942,
|
| 1699 |
+
"rewards/margins_std": 0.08615661412477493,
|
| 1700 |
+
"rewards/rejected": -0.014344056136906147,
|
| 1701 |
+
"step": 930
|
| 1702 |
+
},
|
| 1703 |
+
{
|
| 1704 |
+
"epoch": 0.69,
|
| 1705 |
+
"grad_norm": 0.4453125,
|
| 1706 |
+
"learning_rate": 1.3135514145204606e-07,
|
| 1707 |
+
"logits/chosen": 0.046013571321964264,
|
| 1708 |
+
"logits/rejected": 0.5954084396362305,
|
| 1709 |
+
"logps/chosen": -269.13629150390625,
|
| 1710 |
+
"logps/rejected": -249.99008178710938,
|
| 1711 |
+
"loss": 0.6511,
|
| 1712 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 1713 |
+
"rewards/chosen": 0.06792394816875458,
|
| 1714 |
+
"rewards/margins": 0.08472023159265518,
|
| 1715 |
+
"rewards/margins_max": 0.12758344411849976,
|
| 1716 |
+
"rewards/margins_min": 0.0418570414185524,
|
| 1717 |
+
"rewards/margins_std": 0.06061771512031555,
|
| 1718 |
+
"rewards/rejected": -0.016796285286545753,
|
| 1719 |
+
"step": 940
|
| 1720 |
+
},
|
| 1721 |
+
{
|
| 1722 |
+
"epoch": 0.7,
|
| 1723 |
+
"grad_norm": 0.515625,
|
| 1724 |
+
"learning_rate": 1.257422686625539e-07,
|
| 1725 |
+
"logits/chosen": -0.12281881272792816,
|
| 1726 |
+
"logits/rejected": 0.7430060505867004,
|
| 1727 |
+
"logps/chosen": -328.2705383300781,
|
| 1728 |
+
"logps/rejected": -239.69070434570312,
|
| 1729 |
+
"loss": 0.6552,
|
| 1730 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 1731 |
+
"rewards/chosen": 0.06796546280384064,
|
| 1732 |
+
"rewards/margins": 0.08045408874750137,
|
| 1733 |
+
"rewards/margins_max": 0.1251390427350998,
|
| 1734 |
+
"rewards/margins_min": 0.03576914221048355,
|
| 1735 |
+
"rewards/margins_std": 0.06319407373666763,
|
| 1736 |
+
"rewards/rejected": -0.01248864270746708,
|
| 1737 |
+
"step": 950
|
| 1738 |
+
},
|
| 1739 |
+
{
|
| 1740 |
+
"epoch": 0.71,
|
| 1741 |
+
"grad_norm": 0.49609375,
|
| 1742 |
+
"learning_rate": 1.2021138313265444e-07,
|
| 1743 |
+
"logits/chosen": -0.05693725496530533,
|
| 1744 |
+
"logits/rejected": 0.5191583633422852,
|
| 1745 |
+
"logps/chosen": -267.0467224121094,
|
| 1746 |
+
"logps/rejected": -253.3563995361328,
|
| 1747 |
+
"loss": 0.6547,
|
| 1748 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 1749 |
+
"rewards/chosen": 0.06574182212352753,
|
| 1750 |
+
"rewards/margins": 0.0813496932387352,
|
| 1751 |
+
"rewards/margins_max": 0.11537833511829376,
|
| 1752 |
+
"rewards/margins_min": 0.04732104018330574,
|
| 1753 |
+
"rewards/margins_std": 0.048123784363269806,
|
| 1754 |
+
"rewards/rejected": -0.015607871115207672,
|
| 1755 |
+
"step": 960
|
| 1756 |
+
},
|
| 1757 |
+
{
|
| 1758 |
+
"epoch": 0.71,
|
| 1759 |
+
"grad_norm": 0.421875,
|
| 1760 |
+
"learning_rate": 1.1476613423001974e-07,
|
| 1761 |
+
"logits/chosen": 0.01074630580842495,
|
| 1762 |
+
"logits/rejected": 0.7058780789375305,
|
| 1763 |
+
"logps/chosen": -268.10162353515625,
|
| 1764 |
+
"logps/rejected": -246.88095092773438,
|
| 1765 |
+
"loss": 0.6503,
|
| 1766 |
+
"rewards/accuracies": 0.925000011920929,
|
| 1767 |
+
"rewards/chosen": 0.06480704247951508,
|
| 1768 |
+
"rewards/margins": 0.07971958816051483,
|
| 1769 |
+
"rewards/margins_max": 0.12575621902942657,
|
| 1770 |
+
"rewards/margins_min": 0.03368296101689339,
|
| 1771 |
+
"rewards/margins_std": 0.0651056244969368,
|
| 1772 |
+
"rewards/rejected": -0.014912550337612629,
|
| 1773 |
+
"step": 970
|
| 1774 |
+
},
|
| 1775 |
+
{
|
| 1776 |
+
"epoch": 0.72,
|
| 1777 |
+
"grad_norm": 0.447265625,
|
| 1778 |
+
"learning_rate": 1.0941011481789042e-07,
|
| 1779 |
+
"logits/chosen": 0.16233515739440918,
|
| 1780 |
+
"logits/rejected": 0.6619356274604797,
|
| 1781 |
+
"logps/chosen": -268.57537841796875,
|
| 1782 |
+
"logps/rejected": -248.046142578125,
|
| 1783 |
+
"loss": 0.6547,
|
| 1784 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 1785 |
+
"rewards/chosen": 0.061273593455553055,
|
| 1786 |
+
"rewards/margins": 0.07566945254802704,
|
| 1787 |
+
"rewards/margins_max": 0.1124287024140358,
|
| 1788 |
+
"rewards/margins_min": 0.03891021013259888,
|
| 1789 |
+
"rewards/margins_std": 0.051985424011945724,
|
| 1790 |
+
"rewards/rejected": -0.014395867474377155,
|
| 1791 |
+
"step": 980
|
| 1792 |
+
},
|
| 1793 |
+
{
|
| 1794 |
+
"epoch": 0.73,
|
| 1795 |
+
"grad_norm": 0.4296875,
|
| 1796 |
+
"learning_rate": 1.041468588844476e-07,
|
| 1797 |
+
"logits/chosen": 0.021765699610114098,
|
| 1798 |
+
"logits/rejected": 0.5723878145217896,
|
| 1799 |
+
"logps/chosen": -252.99365234375,
|
| 1800 |
+
"logps/rejected": -237.05233764648438,
|
| 1801 |
+
"loss": 0.6506,
|
| 1802 |
+
"rewards/accuracies": 0.925000011920929,
|
| 1803 |
+
"rewards/chosen": 0.06569187343120575,
|
| 1804 |
+
"rewards/margins": 0.08357974141836166,
|
| 1805 |
+
"rewards/margins_max": 0.12318392097949982,
|
| 1806 |
+
"rewards/margins_min": 0.043975573033094406,
|
| 1807 |
+
"rewards/margins_std": 0.056008756160736084,
|
| 1808 |
+
"rewards/rejected": -0.017887871712446213,
|
| 1809 |
+
"step": 990
|
| 1810 |
+
},
|
| 1811 |
+
{
|
| 1812 |
+
"epoch": 0.74,
|
| 1813 |
+
"grad_norm": 0.474609375,
|
| 1814 |
+
"learning_rate": 9.897983921102954e-08,
|
| 1815 |
+
"logits/chosen": -0.08432263880968094,
|
| 1816 |
+
"logits/rejected": 0.5865111351013184,
|
| 1817 |
+
"logps/chosen": -265.72735595703125,
|
| 1818 |
+
"logps/rejected": -234.95028686523438,
|
| 1819 |
+
"loss": 0.6572,
|
| 1820 |
+
"rewards/accuracies": 0.949999988079071,
|
| 1821 |
+
"rewards/chosen": 0.060374390333890915,
|
| 1822 |
+
"rewards/margins": 0.070238396525383,
|
| 1823 |
+
"rewards/margins_max": 0.10305947065353394,
|
| 1824 |
+
"rewards/margins_min": 0.037417326122522354,
|
| 1825 |
+
"rewards/margins_std": 0.04641600325703621,
|
| 1826 |
+
"rewards/rejected": -0.009864002466201782,
|
| 1827 |
+
"step": 1000
|
| 1828 |
+
},
|
| 1829 |
+
{
|
| 1830 |
+
"epoch": 0.74,
|
| 1831 |
+
"grad_norm": 0.42578125,
|
| 1832 |
+
"learning_rate": 9.391246508073433e-08,
|
| 1833 |
+
"logits/chosen": -0.17238160967826843,
|
| 1834 |
+
"logits/rejected": 0.6868064999580383,
|
| 1835 |
+
"logps/chosen": -321.6485595703125,
|
| 1836 |
+
"logps/rejected": -263.62188720703125,
|
| 1837 |
+
"loss": 0.6505,
|
| 1838 |
+
"rewards/accuracies": 0.9375,
|
| 1839 |
+
"rewards/chosen": 0.07812955230474472,
|
| 1840 |
+
"rewards/margins": 0.09409363567829132,
|
| 1841 |
+
"rewards/margins_max": 0.13336913287639618,
|
| 1842 |
+
"rewards/margins_min": 0.054818131029605865,
|
| 1843 |
+
"rewards/margins_std": 0.05554394796490669,
|
| 1844 |
+
"rewards/rejected": -0.015964074060320854,
|
| 1845 |
+
"step": 1010
|
| 1846 |
+
},
|
| 1847 |
+
{
|
| 1848 |
+
"epoch": 0.75,
|
| 1849 |
+
"grad_norm": 0.408203125,
|
| 1850 |
+
"learning_rate": 8.894808002892037e-08,
|
| 1851 |
+
"logits/chosen": -0.048636335879564285,
|
| 1852 |
+
"logits/rejected": 0.6718664765357971,
|
| 1853 |
+
"logps/chosen": -286.724365234375,
|
| 1854 |
+
"logps/rejected": -256.17767333984375,
|
| 1855 |
+
"loss": 0.6521,
|
| 1856 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 1857 |
+
"rewards/chosen": 0.07918758690357208,
|
| 1858 |
+
"rewards/margins": 0.09503775835037231,
|
| 1859 |
+
"rewards/margins_max": 0.1390329748392105,
|
| 1860 |
+
"rewards/margins_min": 0.05104254558682442,
|
| 1861 |
+
"rewards/margins_std": 0.06221862882375717,
|
| 1862 |
+
"rewards/rejected": -0.015850163996219635,
|
| 1863 |
+
"step": 1020
|
| 1864 |
+
},
|
| 1865 |
+
{
|
| 1866 |
+
"epoch": 0.76,
|
| 1867 |
+
"grad_norm": 0.423828125,
|
| 1868 |
+
"learning_rate": 8.408995963708756e-08,
|
| 1869 |
+
"logits/chosen": -0.03906983509659767,
|
| 1870 |
+
"logits/rejected": 0.7512571215629578,
|
| 1871 |
+
"logps/chosen": -289.33660888671875,
|
| 1872 |
+
"logps/rejected": -260.1453857421875,
|
| 1873 |
+
"loss": 0.6546,
|
| 1874 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 1875 |
+
"rewards/chosen": 0.07737907022237778,
|
| 1876 |
+
"rewards/margins": 0.08742411434650421,
|
| 1877 |
+
"rewards/margins_max": 0.12180934101343155,
|
| 1878 |
+
"rewards/margins_min": 0.053038887679576874,
|
| 1879 |
+
"rewards/margins_std": 0.048628050833940506,
|
| 1880 |
+
"rewards/rejected": -0.010045038536190987,
|
| 1881 |
+
"step": 1030
|
| 1882 |
+
},
|
| 1883 |
+
{
|
| 1884 |
+
"epoch": 0.77,
|
| 1885 |
+
"grad_norm": 0.484375,
|
| 1886 |
+
"learning_rate": 7.934130937159508e-08,
|
| 1887 |
+
"logits/chosen": 0.0004897325998172164,
|
| 1888 |
+
"logits/rejected": 0.7238715887069702,
|
| 1889 |
+
"logps/chosen": -279.21624755859375,
|
| 1890 |
+
"logps/rejected": -248.09957885742188,
|
| 1891 |
+
"loss": 0.6519,
|
| 1892 |
+
"rewards/accuracies": 0.987500011920929,
|
| 1893 |
+
"rewards/chosen": 0.07424648851156235,
|
| 1894 |
+
"rewards/margins": 0.08449728786945343,
|
| 1895 |
+
"rewards/margins_max": 0.1293594241142273,
|
| 1896 |
+
"rewards/margins_min": 0.039635173976421356,
|
| 1897 |
+
"rewards/margins_std": 0.06344461441040039,
|
| 1898 |
+
"rewards/rejected": -0.010250799357891083,
|
| 1899 |
+
"step": 1040
|
| 1900 |
+
},
|
| 1901 |
+
{
|
| 1902 |
+
"epoch": 0.77,
|
| 1903 |
+
"grad_norm": 0.455078125,
|
| 1904 |
+
"learning_rate": 7.470526246864364e-08,
|
| 1905 |
+
"logits/chosen": -0.028243955224752426,
|
| 1906 |
+
"logits/rejected": 0.6930115818977356,
|
| 1907 |
+
"logps/chosen": -301.51458740234375,
|
| 1908 |
+
"logps/rejected": -251.8220672607422,
|
| 1909 |
+
"loss": 0.6559,
|
| 1910 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 1911 |
+
"rewards/chosen": 0.06157411262392998,
|
| 1912 |
+
"rewards/margins": 0.07462558895349503,
|
| 1913 |
+
"rewards/margins_max": 0.10729455947875977,
|
| 1914 |
+
"rewards/margins_min": 0.04195661470293999,
|
| 1915 |
+
"rewards/margins_std": 0.04620090872049332,
|
| 1916 |
+
"rewards/rejected": -0.01305147260427475,
|
| 1917 |
+
"step": 1050
|
| 1918 |
+
},
|
| 1919 |
+
{
|
| 1920 |
+
"epoch": 0.78,
|
| 1921 |
+
"grad_norm": 0.494140625,
|
| 1922 |
+
"learning_rate": 7.018487786691512e-08,
|
| 1923 |
+
"logits/chosen": -0.18514272570610046,
|
| 1924 |
+
"logits/rejected": 0.8259115219116211,
|
| 1925 |
+
"logps/chosen": -284.2102355957031,
|
| 1926 |
+
"logps/rejected": -236.52206420898438,
|
| 1927 |
+
"loss": 0.6474,
|
| 1928 |
+
"rewards/accuracies": 0.987500011920929,
|
| 1929 |
+
"rewards/chosen": 0.09142851829528809,
|
| 1930 |
+
"rewards/margins": 0.10867790132761002,
|
| 1931 |
+
"rewards/margins_max": 0.16026386618614197,
|
| 1932 |
+
"rewards/margins_min": 0.05709192156791687,
|
| 1933 |
+
"rewards/margins_std": 0.07295358926057816,
|
| 1934 |
+
"rewards/rejected": -0.01724938116967678,
|
| 1935 |
+
"step": 1060
|
| 1936 |
+
},
|
| 1937 |
+
{
|
| 1938 |
+
"epoch": 0.79,
|
| 1939 |
+
"grad_norm": 0.4296875,
|
| 1940 |
+
"learning_rate": 6.578313818923559e-08,
|
| 1941 |
+
"logits/chosen": 0.12218723446130753,
|
| 1942 |
+
"logits/rejected": 0.6379785537719727,
|
| 1943 |
+
"logps/chosen": -238.366943359375,
|
| 1944 |
+
"logps/rejected": -233.61026000976562,
|
| 1945 |
+
"loss": 0.6535,
|
| 1946 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 1947 |
+
"rewards/chosen": 0.055001966655254364,
|
| 1948 |
+
"rewards/margins": 0.06915679574012756,
|
| 1949 |
+
"rewards/margins_max": 0.09845878183841705,
|
| 1950 |
+
"rewards/margins_min": 0.039854805916547775,
|
| 1951 |
+
"rewards/margins_std": 0.04143926501274109,
|
| 1952 |
+
"rewards/rejected": -0.014154826290905476,
|
| 1953 |
+
"step": 1070
|
| 1954 |
+
},
|
| 1955 |
+
{
|
| 1956 |
+
"epoch": 0.79,
|
| 1957 |
+
"grad_norm": 0.40234375,
|
| 1958 |
+
"learning_rate": 6.15029477745925e-08,
|
| 1959 |
+
"logits/chosen": -0.1947169005870819,
|
| 1960 |
+
"logits/rejected": 0.7179273366928101,
|
| 1961 |
+
"logps/chosen": -295.55096435546875,
|
| 1962 |
+
"logps/rejected": -222.1858367919922,
|
| 1963 |
+
"loss": 0.6512,
|
| 1964 |
+
"rewards/accuracies": 0.9375,
|
| 1965 |
+
"rewards/chosen": 0.07841251790523529,
|
| 1966 |
+
"rewards/margins": 0.08169189840555191,
|
| 1967 |
+
"rewards/margins_max": 0.11758895218372345,
|
| 1968 |
+
"rewards/margins_min": 0.045794855803251266,
|
| 1969 |
+
"rewards/margins_std": 0.05076608806848526,
|
| 1970 |
+
"rewards/rejected": -0.003279387950897217,
|
| 1971 |
+
"step": 1080
|
| 1972 |
+
},
|
| 1973 |
+
{
|
| 1974 |
+
"epoch": 0.8,
|
| 1975 |
+
"grad_norm": 0.44921875,
|
| 1976 |
+
"learning_rate": 5.734713076180486e-08,
|
| 1977 |
+
"logits/chosen": 0.006981179118156433,
|
| 1978 |
+
"logits/rejected": 0.7452491521835327,
|
| 1979 |
+
"logps/chosen": -260.1736145019531,
|
| 1980 |
+
"logps/rejected": -224.355224609375,
|
| 1981 |
+
"loss": 0.6557,
|
| 1982 |
+
"rewards/accuracies": 0.987500011920929,
|
| 1983 |
+
"rewards/chosen": 0.06287099421024323,
|
| 1984 |
+
"rewards/margins": 0.07665695995092392,
|
| 1985 |
+
"rewards/margins_max": 0.11319824308156967,
|
| 1986 |
+
"rewards/margins_min": 0.04011568799614906,
|
| 1987 |
+
"rewards/margins_std": 0.05167717486619949,
|
| 1988 |
+
"rewards/rejected": -0.013785977847874165,
|
| 1989 |
+
"step": 1090
|
| 1990 |
+
},
|
| 1991 |
+
{
|
| 1992 |
+
"epoch": 0.81,
|
| 1993 |
+
"grad_norm": 0.48046875,
|
| 1994 |
+
"learning_rate": 5.3318429226110875e-08,
|
| 1995 |
+
"logits/chosen": 0.015549438074231148,
|
| 1996 |
+
"logits/rejected": 0.6923533082008362,
|
| 1997 |
+
"logps/chosen": -269.09747314453125,
|
| 1998 |
+
"logps/rejected": -236.7939910888672,
|
| 1999 |
+
"loss": 0.6512,
|
| 2000 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 2001 |
+
"rewards/chosen": 0.06426068395376205,
|
| 2002 |
+
"rewards/margins": 0.08864103257656097,
|
| 2003 |
+
"rewards/margins_max": 0.12798473238945007,
|
| 2004 |
+
"rewards/margins_min": 0.04929735139012337,
|
| 2005 |
+
"rewards/margins_std": 0.05564037710428238,
|
| 2006 |
+
"rewards/rejected": -0.024380357936024666,
|
| 2007 |
+
"step": 1100
|
| 2008 |
+
},
|
| 2009 |
+
{
|
| 2010 |
+
"epoch": 0.82,
|
| 2011 |
+
"grad_norm": 0.419921875,
|
| 2012 |
+
"learning_rate": 4.9419501369902026e-08,
|
| 2013 |
+
"logits/chosen": -0.029879633337259293,
|
| 2014 |
+
"logits/rejected": 0.7326146960258484,
|
| 2015 |
+
"logps/chosen": -260.98968505859375,
|
| 2016 |
+
"logps/rejected": -220.9407958984375,
|
| 2017 |
+
"loss": 0.6499,
|
| 2018 |
+
"rewards/accuracies": 0.949999988079071,
|
| 2019 |
+
"rewards/chosen": 0.07826592028141022,
|
| 2020 |
+
"rewards/margins": 0.09226210415363312,
|
| 2021 |
+
"rewards/margins_max": 0.1359224021434784,
|
| 2022 |
+
"rewards/margins_min": 0.04860180616378784,
|
| 2023 |
+
"rewards/margins_std": 0.06174498051404953,
|
| 2024 |
+
"rewards/rejected": -0.0139961838722229,
|
| 2025 |
+
"step": 1110
|
| 2026 |
+
},
|
| 2027 |
+
{
|
| 2028 |
+
"epoch": 0.82,
|
| 2029 |
+
"grad_norm": 0.42578125,
|
| 2030 |
+
"learning_rate": 4.5652919768798896e-08,
|
| 2031 |
+
"logits/chosen": -0.1553410142660141,
|
| 2032 |
+
"logits/rejected": 0.6449904441833496,
|
| 2033 |
+
"logps/chosen": -280.3263244628906,
|
| 2034 |
+
"logps/rejected": -242.4776153564453,
|
| 2035 |
+
"loss": 0.6512,
|
| 2036 |
+
"rewards/accuracies": 0.987500011920929,
|
| 2037 |
+
"rewards/chosen": 0.074976846575737,
|
| 2038 |
+
"rewards/margins": 0.08818279206752777,
|
| 2039 |
+
"rewards/margins_max": 0.12042693048715591,
|
| 2040 |
+
"rewards/margins_min": 0.05593867227435112,
|
| 2041 |
+
"rewards/margins_std": 0.04560008645057678,
|
| 2042 |
+
"rewards/rejected": -0.013205950148403645,
|
| 2043 |
+
"step": 1120
|
| 2044 |
+
},
|
| 2045 |
+
{
|
| 2046 |
+
"epoch": 0.83,
|
| 2047 |
+
"grad_norm": 0.474609375,
|
| 2048 |
+
"learning_rate": 4.2021169674223536e-08,
|
| 2049 |
+
"logits/chosen": -0.12581007182598114,
|
| 2050 |
+
"logits/rejected": 0.6627474427223206,
|
| 2051 |
+
"logps/chosen": -274.0786437988281,
|
| 2052 |
+
"logps/rejected": -250.4473114013672,
|
| 2053 |
+
"loss": 0.6483,
|
| 2054 |
+
"rewards/accuracies": 0.987500011920929,
|
| 2055 |
+
"rewards/chosen": 0.0676179900765419,
|
| 2056 |
+
"rewards/margins": 0.08669795840978622,
|
| 2057 |
+
"rewards/margins_max": 0.12913702428340912,
|
| 2058 |
+
"rewards/margins_min": 0.04425888508558273,
|
| 2059 |
+
"rewards/margins_std": 0.060017913579940796,
|
| 2060 |
+
"rewards/rejected": -0.01907997578382492,
|
| 2061 |
+
"step": 1130
|
| 2062 |
+
},
|
| 2063 |
+
{
|
| 2064 |
+
"epoch": 0.84,
|
| 2065 |
+
"grad_norm": 0.369140625,
|
| 2066 |
+
"learning_rate": 3.852664737359046e-08,
|
| 2067 |
+
"logits/chosen": 0.08468042314052582,
|
| 2068 |
+
"logits/rejected": 0.7667611241340637,
|
| 2069 |
+
"logps/chosen": -249.0133819580078,
|
| 2070 |
+
"logps/rejected": -232.16561889648438,
|
| 2071 |
+
"loss": 0.6497,
|
| 2072 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 2073 |
+
"rewards/chosen": 0.07480134069919586,
|
| 2074 |
+
"rewards/margins": 0.09523622691631317,
|
| 2075 |
+
"rewards/margins_max": 0.15331092476844788,
|
| 2076 |
+
"rewards/margins_min": 0.037161558866500854,
|
| 2077 |
+
"rewards/margins_std": 0.08212999999523163,
|
| 2078 |
+
"rewards/rejected": -0.020434897392988205,
|
| 2079 |
+
"step": 1140
|
| 2080 |
+
},
|
| 2081 |
+
{
|
| 2082 |
+
"epoch": 0.85,
|
| 2083 |
+
"grad_norm": 0.373046875,
|
| 2084 |
+
"learning_rate": 3.5171658609197824e-08,
|
| 2085 |
+
"logits/chosen": -0.03485478088259697,
|
| 2086 |
+
"logits/rejected": 0.5982181429862976,
|
| 2087 |
+
"logps/chosen": -252.3005828857422,
|
| 2088 |
+
"logps/rejected": -240.3730926513672,
|
| 2089 |
+
"loss": 0.6515,
|
| 2090 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 2091 |
+
"rewards/chosen": 0.06789231300354004,
|
| 2092 |
+
"rewards/margins": 0.0819520652294159,
|
| 2093 |
+
"rewards/margins_max": 0.11909864842891693,
|
| 2094 |
+
"rewards/margins_min": 0.04480547830462456,
|
| 2095 |
+
"rewards/margins_std": 0.05253320187330246,
|
| 2096 |
+
"rewards/rejected": -0.014059755019843578,
|
| 2097 |
+
"step": 1150
|
| 2098 |
+
},
|
| 2099 |
+
{
|
| 2100 |
+
"epoch": 0.85,
|
| 2101 |
+
"grad_norm": 0.435546875,
|
| 2102 |
+
"learning_rate": 3.195841705686139e-08,
|
| 2103 |
+
"logits/chosen": 0.001758894301019609,
|
| 2104 |
+
"logits/rejected": 0.6828774213790894,
|
| 2105 |
+
"logps/chosen": -259.75640869140625,
|
| 2106 |
+
"logps/rejected": -243.09970092773438,
|
| 2107 |
+
"loss": 0.6517,
|
| 2108 |
+
"rewards/accuracies": 0.987500011920929,
|
| 2109 |
+
"rewards/chosen": 0.06515143811702728,
|
| 2110 |
+
"rewards/margins": 0.07973488420248032,
|
| 2111 |
+
"rewards/margins_max": 0.11704728752374649,
|
| 2112 |
+
"rewards/margins_min": 0.042422495782375336,
|
| 2113 |
+
"rewards/margins_std": 0.05276770144701004,
|
| 2114 |
+
"rewards/rejected": -0.014583444222807884,
|
| 2115 |
+
"step": 1160
|
| 2116 |
+
},
|
| 2117 |
+
{
|
| 2118 |
+
"epoch": 0.86,
|
| 2119 |
+
"grad_norm": 0.4375,
|
| 2120 |
+
"learning_rate": 2.8889042865294837e-08,
|
| 2121 |
+
"logits/chosen": 0.0009852380026131868,
|
| 2122 |
+
"logits/rejected": 0.7770395278930664,
|
| 2123 |
+
"logps/chosen": -252.7977752685547,
|
| 2124 |
+
"logps/rejected": -211.03494262695312,
|
| 2125 |
+
"loss": 0.652,
|
| 2126 |
+
"rewards/accuracies": 0.949999988079071,
|
| 2127 |
+
"rewards/chosen": 0.06879086047410965,
|
| 2128 |
+
"rewards/margins": 0.0823785737156868,
|
| 2129 |
+
"rewards/margins_max": 0.126399964094162,
|
| 2130 |
+
"rewards/margins_min": 0.038357168436050415,
|
| 2131 |
+
"rewards/margins_std": 0.06225566938519478,
|
| 2132 |
+
"rewards/rejected": -0.013587711378932,
|
| 2133 |
+
"step": 1170
|
| 2134 |
+
},
|
| 2135 |
+
{
|
| 2136 |
+
"epoch": 0.87,
|
| 2137 |
+
"grad_norm": 0.5234375,
|
| 2138 |
+
"learning_rate": 2.5965561257202036e-08,
|
| 2139 |
+
"logits/chosen": -0.0783575028181076,
|
| 2140 |
+
"logits/rejected": 0.7545520067214966,
|
| 2141 |
+
"logps/chosen": -272.24041748046875,
|
| 2142 |
+
"logps/rejected": -247.2455291748047,
|
| 2143 |
+
"loss": 0.6473,
|
| 2144 |
+
"rewards/accuracies": 0.949999988079071,
|
| 2145 |
+
"rewards/chosen": 0.09078168123960495,
|
| 2146 |
+
"rewards/margins": 0.10268386453390121,
|
| 2147 |
+
"rewards/margins_max": 0.14412382245063782,
|
| 2148 |
+
"rewards/margins_min": 0.06124391406774521,
|
| 2149 |
+
"rewards/margins_std": 0.05860493704676628,
|
| 2150 |
+
"rewards/rejected": -0.011902175843715668,
|
| 2151 |
+
"step": 1180
|
| 2152 |
+
},
|
| 2153 |
+
{
|
| 2154 |
+
"epoch": 0.88,
|
| 2155 |
+
"grad_norm": 0.80078125,
|
| 2156 |
+
"learning_rate": 2.318990119300218e-08,
|
| 2157 |
+
"logits/chosen": 0.05255628377199173,
|
| 2158 |
+
"logits/rejected": 0.6947780847549438,
|
| 2159 |
+
"logps/chosen": -243.50509643554688,
|
| 2160 |
+
"logps/rejected": -212.8872528076172,
|
| 2161 |
+
"loss": 0.6546,
|
| 2162 |
+
"rewards/accuracies": 0.9375,
|
| 2163 |
+
"rewards/chosen": 0.05941227823495865,
|
| 2164 |
+
"rewards/margins": 0.07317686825990677,
|
| 2165 |
+
"rewards/margins_max": 0.11058169603347778,
|
| 2166 |
+
"rewards/margins_min": 0.03577204793691635,
|
| 2167 |
+
"rewards/margins_std": 0.052898406982421875,
|
| 2168 |
+
"rewards/rejected": -0.013764592818915844,
|
| 2169 |
+
"step": 1190
|
| 2170 |
+
},
|
| 2171 |
+
{
|
| 2172 |
+
"epoch": 0.88,
|
| 2173 |
+
"grad_norm": 0.4296875,
|
| 2174 |
+
"learning_rate": 2.0563894098070216e-08,
|
| 2175 |
+
"logits/chosen": -0.0692586898803711,
|
| 2176 |
+
"logits/rejected": 0.7870053052902222,
|
| 2177 |
+
"logps/chosen": -309.2232666015625,
|
| 2178 |
+
"logps/rejected": -266.5531921386719,
|
| 2179 |
+
"loss": 0.6476,
|
| 2180 |
+
"rewards/accuracies": 0.925000011920929,
|
| 2181 |
+
"rewards/chosen": 0.08217885345220566,
|
| 2182 |
+
"rewards/margins": 0.09460541605949402,
|
| 2183 |
+
"rewards/margins_max": 0.13726098835468292,
|
| 2184 |
+
"rewards/margins_min": 0.05194983631372452,
|
| 2185 |
+
"rewards/margins_std": 0.06032410264015198,
|
| 2186 |
+
"rewards/rejected": -0.012426558881998062,
|
| 2187 |
+
"step": 1200
|
| 2188 |
+
},
|
| 2189 |
+
{
|
| 2190 |
+
"epoch": 0.89,
|
| 2191 |
+
"grad_norm": 0.34375,
|
| 2192 |
+
"learning_rate": 1.8089272654333353e-08,
|
| 2193 |
+
"logits/chosen": -0.046720363199710846,
|
| 2194 |
+
"logits/rejected": 0.6504586338996887,
|
| 2195 |
+
"logps/chosen": -261.49298095703125,
|
| 2196 |
+
"logps/rejected": -223.7759552001953,
|
| 2197 |
+
"loss": 0.6526,
|
| 2198 |
+
"rewards/accuracies": 0.9375,
|
| 2199 |
+
"rewards/chosen": 0.07228533923625946,
|
| 2200 |
+
"rewards/margins": 0.07905267179012299,
|
| 2201 |
+
"rewards/margins_max": 0.12283986806869507,
|
| 2202 |
+
"rewards/margins_min": 0.035265471786260605,
|
| 2203 |
+
"rewards/margins_std": 0.06192445755004883,
|
| 2204 |
+
"rewards/rejected": -0.006767333950847387,
|
| 2205 |
+
"step": 1210
|
| 2206 |
+
},
|
| 2207 |
+
{
|
| 2208 |
+
"epoch": 0.9,
|
| 2209 |
+
"grad_norm": 0.4140625,
|
| 2210 |
+
"learning_rate": 1.5767669657019005e-08,
|
| 2211 |
+
"logits/chosen": -0.1003817468881607,
|
| 2212 |
+
"logits/rejected": 0.7918148636817932,
|
| 2213 |
+
"logps/chosen": -284.4376525878906,
|
| 2214 |
+
"logps/rejected": -226.739990234375,
|
| 2215 |
+
"loss": 0.6514,
|
| 2216 |
+
"rewards/accuracies": 0.949999988079071,
|
| 2217 |
+
"rewards/chosen": 0.07430224120616913,
|
| 2218 |
+
"rewards/margins": 0.08967499434947968,
|
| 2219 |
+
"rewards/margins_max": 0.13941441476345062,
|
| 2220 |
+
"rewards/margins_min": 0.03993559628725052,
|
| 2221 |
+
"rewards/margins_std": 0.07034213840961456,
|
| 2222 |
+
"rewards/rejected": -0.015372763387858868,
|
| 2223 |
+
"step": 1220
|
| 2224 |
+
},
|
| 2225 |
+
{
|
| 2226 |
+
"epoch": 0.91,
|
| 2227 |
+
"grad_norm": 0.42578125,
|
| 2228 |
+
"learning_rate": 1.3600616937310267e-08,
|
| 2229 |
+
"logits/chosen": -0.13235566020011902,
|
| 2230 |
+
"logits/rejected": 0.602156400680542,
|
| 2231 |
+
"logps/chosen": -261.45050048828125,
|
| 2232 |
+
"logps/rejected": -231.56277465820312,
|
| 2233 |
+
"loss": 0.647,
|
| 2234 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 2235 |
+
"rewards/chosen": 0.0806741714477539,
|
| 2236 |
+
"rewards/margins": 0.10508724302053452,
|
| 2237 |
+
"rewards/margins_max": 0.1610412895679474,
|
| 2238 |
+
"rewards/margins_min": 0.049133218824863434,
|
| 2239 |
+
"rewards/margins_std": 0.07913094758987427,
|
| 2240 |
+
"rewards/rejected": -0.024413082748651505,
|
| 2241 |
+
"step": 1230
|
| 2242 |
+
},
|
| 2243 |
+
{
|
| 2244 |
+
"epoch": 0.91,
|
| 2245 |
+
"grad_norm": 0.478515625,
|
| 2246 |
+
"learning_rate": 1.1589544351619047e-08,
|
| 2247 |
+
"logits/chosen": 0.02703939750790596,
|
| 2248 |
+
"logits/rejected": 0.8025895357131958,
|
| 2249 |
+
"logps/chosen": -334.37176513671875,
|
| 2250 |
+
"logps/rejected": -263.4154968261719,
|
| 2251 |
+
"loss": 0.6488,
|
| 2252 |
+
"rewards/accuracies": 0.949999988079071,
|
| 2253 |
+
"rewards/chosen": 0.07323239743709564,
|
| 2254 |
+
"rewards/margins": 0.08194652944803238,
|
| 2255 |
+
"rewards/margins_max": 0.12296142429113388,
|
| 2256 |
+
"rewards/margins_min": 0.04093163460493088,
|
| 2257 |
+
"rewards/margins_std": 0.05800382047891617,
|
| 2258 |
+
"rewards/rejected": -0.008714130148291588,
|
| 2259 |
+
"step": 1240
|
| 2260 |
+
},
|
| 2261 |
+
{
|
| 2262 |
+
"epoch": 0.92,
|
| 2263 |
+
"grad_norm": 0.359375,
|
| 2264 |
+
"learning_rate": 9.735778838143749e-09,
|
| 2265 |
+
"logits/chosen": -0.0286453515291214,
|
| 2266 |
+
"logits/rejected": 0.7510533332824707,
|
| 2267 |
+
"logps/chosen": -311.01959228515625,
|
| 2268 |
+
"logps/rejected": -240.82870483398438,
|
| 2269 |
+
"loss": 0.6492,
|
| 2270 |
+
"rewards/accuracies": 0.987500011920929,
|
| 2271 |
+
"rewards/chosen": 0.07665716856718063,
|
| 2272 |
+
"rewards/margins": 0.09343422949314117,
|
| 2273 |
+
"rewards/margins_max": 0.13625505566596985,
|
| 2274 |
+
"rewards/margins_min": 0.0506133958697319,
|
| 2275 |
+
"rewards/margins_std": 0.06055780500173569,
|
| 2276 |
+
"rewards/rejected": -0.01677705906331539,
|
| 2277 |
+
"step": 1250
|
| 2278 |
+
},
|
| 2279 |
+
{
|
| 2280 |
+
"epoch": 0.93,
|
| 2281 |
+
"grad_norm": 0.44140625,
|
| 2282 |
+
"learning_rate": 8.040543541333655e-09,
|
| 2283 |
+
"logits/chosen": 0.05244514346122742,
|
| 2284 |
+
"logits/rejected": 0.7751520276069641,
|
| 2285 |
+
"logps/chosen": -310.7460021972656,
|
| 2286 |
+
"logps/rejected": -286.03826904296875,
|
| 2287 |
+
"loss": 0.6489,
|
| 2288 |
+
"rewards/accuracies": 0.987500011920929,
|
| 2289 |
+
"rewards/chosen": 0.06985687464475632,
|
| 2290 |
+
"rewards/margins": 0.0900772362947464,
|
| 2291 |
+
"rewards/margins_max": 0.12786877155303955,
|
| 2292 |
+
"rewards/margins_min": 0.052285682410001755,
|
| 2293 |
+
"rewards/margins_std": 0.053445328027009964,
|
| 2294 |
+
"rewards/rejected": -0.020220357924699783,
|
| 2295 |
+
"step": 1260
|
| 2296 |
+
},
|
| 2297 |
+
{
|
| 2298 |
+
"epoch": 0.93,
|
| 2299 |
+
"grad_norm": 0.5,
|
| 2300 |
+
"learning_rate": 6.504957004838746e-09,
|
| 2301 |
+
"logits/chosen": -0.03165416046977043,
|
| 2302 |
+
"logits/rejected": 0.822761058807373,
|
| 2303 |
+
"logps/chosen": -310.04400634765625,
|
| 2304 |
+
"logps/rejected": -237.48782348632812,
|
| 2305 |
+
"loss": 0.6523,
|
| 2306 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 2307 |
+
"rewards/chosen": 0.06924634426832199,
|
| 2308 |
+
"rewards/margins": 0.08929922431707382,
|
| 2309 |
+
"rewards/margins_max": 0.14400045573711395,
|
| 2310 |
+
"rewards/margins_min": 0.034597981721162796,
|
| 2311 |
+
"rewards/margins_std": 0.07735923677682877,
|
| 2312 |
+
"rewards/rejected": -0.020052876323461533,
|
| 2313 |
+
"step": 1270
|
| 2314 |
+
},
|
| 2315 |
+
{
|
| 2316 |
+
"epoch": 0.94,
|
| 2317 |
+
"grad_norm": 0.4765625,
|
| 2318 |
+
"learning_rate": 5.130032433476483e-09,
|
| 2319 |
+
"logits/chosen": -0.011189639568328857,
|
| 2320 |
+
"logits/rejected": 0.691806972026825,
|
| 2321 |
+
"logps/chosen": -271.3976135253906,
|
| 2322 |
+
"logps/rejected": -269.19390869140625,
|
| 2323 |
+
"loss": 0.6545,
|
| 2324 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 2325 |
+
"rewards/chosen": 0.08424172550439835,
|
| 2326 |
+
"rewards/margins": 0.09692950546741486,
|
| 2327 |
+
"rewards/margins_max": 0.14971934258937836,
|
| 2328 |
+
"rewards/margins_min": 0.044139645993709564,
|
| 2329 |
+
"rewards/margins_std": 0.07465612888336182,
|
| 2330 |
+
"rewards/rejected": -0.012687774375081062,
|
| 2331 |
+
"step": 1280
|
| 2332 |
+
},
|
| 2333 |
+
{
|
| 2334 |
+
"epoch": 0.95,
|
| 2335 |
+
"grad_norm": 0.43359375,
|
| 2336 |
+
"learning_rate": 3.916677024702858e-09,
|
| 2337 |
+
"logits/chosen": 0.009526084177196026,
|
| 2338 |
+
"logits/rejected": 0.7200338244438171,
|
| 2339 |
+
"logps/chosen": -255.1676788330078,
|
| 2340 |
+
"logps/rejected": -234.27432250976562,
|
| 2341 |
+
"loss": 0.6534,
|
| 2342 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 2343 |
+
"rewards/chosen": 0.06782687455415726,
|
| 2344 |
+
"rewards/margins": 0.08498277515172958,
|
| 2345 |
+
"rewards/margins_max": 0.1279430389404297,
|
| 2346 |
+
"rewards/margins_min": 0.04202253371477127,
|
| 2347 |
+
"rewards/margins_std": 0.06075497344136238,
|
| 2348 |
+
"rewards/rejected": -0.017155908048152924,
|
| 2349 |
+
"step": 1290
|
| 2350 |
+
},
|
| 2351 |
+
{
|
| 2352 |
+
"epoch": 0.96,
|
| 2353 |
+
"grad_norm": 0.51953125,
|
| 2354 |
+
"learning_rate": 2.865691370028761e-09,
|
| 2355 |
+
"logits/chosen": -0.11016593128442764,
|
| 2356 |
+
"logits/rejected": 0.7786885499954224,
|
| 2357 |
+
"logps/chosen": -310.4071960449219,
|
| 2358 |
+
"logps/rejected": -242.39505004882812,
|
| 2359 |
+
"loss": 0.6487,
|
| 2360 |
+
"rewards/accuracies": 1.0,
|
| 2361 |
+
"rewards/chosen": 0.0797882154583931,
|
| 2362 |
+
"rewards/margins": 0.09522275626659393,
|
| 2363 |
+
"rewards/margins_max": 0.13675549626350403,
|
| 2364 |
+
"rewards/margins_min": 0.05369000881910324,
|
| 2365 |
+
"rewards/margins_std": 0.0587361678481102,
|
| 2366 |
+
"rewards/rejected": -0.015434542670845985,
|
| 2367 |
+
"step": 1300
|
| 2368 |
+
},
|
| 2369 |
+
{
|
| 2370 |
+
"epoch": 0.96,
|
| 2371 |
+
"grad_norm": 0.482421875,
|
| 2372 |
+
"learning_rate": 1.977768926776896e-09,
|
| 2373 |
+
"logits/chosen": 0.07193834334611893,
|
| 2374 |
+
"logits/rejected": 0.7082736492156982,
|
| 2375 |
+
"logps/chosen": -269.1424255371094,
|
| 2376 |
+
"logps/rejected": -258.5067443847656,
|
| 2377 |
+
"loss": 0.6549,
|
| 2378 |
+
"rewards/accuracies": 0.949999988079071,
|
| 2379 |
+
"rewards/chosen": 0.07826878130435944,
|
| 2380 |
+
"rewards/margins": 0.0938304141163826,
|
| 2381 |
+
"rewards/margins_max": 0.13868080079555511,
|
| 2382 |
+
"rewards/margins_min": 0.04898003116250038,
|
| 2383 |
+
"rewards/margins_std": 0.06342802941799164,
|
| 2384 |
+
"rewards/rejected": -0.01556164026260376,
|
| 2385 |
+
"step": 1310
|
| 2386 |
+
},
|
| 2387 |
+
{
|
| 2388 |
+
"epoch": 0.97,
|
| 2389 |
+
"grad_norm": 0.474609375,
|
| 2390 |
+
"learning_rate": 1.2534955605274233e-09,
|
| 2391 |
+
"logits/chosen": -0.014053803868591785,
|
| 2392 |
+
"logits/rejected": 0.777262806892395,
|
| 2393 |
+
"logps/chosen": -242.13796997070312,
|
| 2394 |
+
"logps/rejected": -218.4778594970703,
|
| 2395 |
+
"loss": 0.6505,
|
| 2396 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 2397 |
+
"rewards/chosen": 0.07983841001987457,
|
| 2398 |
+
"rewards/margins": 0.09085123986005783,
|
| 2399 |
+
"rewards/margins_max": 0.1351737529039383,
|
| 2400 |
+
"rewards/margins_min": 0.046528737992048264,
|
| 2401 |
+
"rewards/margins_std": 0.06268148869276047,
|
| 2402 |
+
"rewards/rejected": -0.011012842878699303,
|
| 2403 |
+
"step": 1320
|
| 2404 |
+
},
|
| 2405 |
+
{
|
| 2406 |
+
"epoch": 0.98,
|
| 2407 |
+
"grad_norm": 0.48828125,
|
| 2408 |
+
"learning_rate": 6.933491585542351e-10,
|
| 2409 |
+
"logits/chosen": 0.03265099227428436,
|
| 2410 |
+
"logits/rejected": 0.7704384326934814,
|
| 2411 |
+
"logps/chosen": -293.1032409667969,
|
| 2412 |
+
"logps/rejected": -263.2946472167969,
|
| 2413 |
+
"loss": 0.6471,
|
| 2414 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 2415 |
+
"rewards/chosen": 0.08791947364807129,
|
| 2416 |
+
"rewards/margins": 0.10431689023971558,
|
| 2417 |
+
"rewards/margins_max": 0.15510039031505585,
|
| 2418 |
+
"rewards/margins_min": 0.05353338643908501,
|
| 2419 |
+
"rewards/margins_std": 0.07181872427463531,
|
| 2420 |
+
"rewards/rejected": -0.016397414728999138,
|
| 2421 |
+
"step": 1330
|
| 2422 |
+
},
|
| 2423 |
+
{
|
| 2424 |
+
"epoch": 0.99,
|
| 2425 |
+
"grad_norm": 0.451171875,
|
| 2426 |
+
"learning_rate": 2.9769931450737694e-10,
|
| 2427 |
+
"logits/chosen": 0.03406764939427376,
|
| 2428 |
+
"logits/rejected": 0.7593650221824646,
|
| 2429 |
+
"logps/chosen": -301.3545227050781,
|
| 2430 |
+
"logps/rejected": -253.1154022216797,
|
| 2431 |
+
"loss": 0.6492,
|
| 2432 |
+
"rewards/accuracies": 0.9624999761581421,
|
| 2433 |
+
"rewards/chosen": 0.06837470084428787,
|
| 2434 |
+
"rewards/margins": 0.08594924956560135,
|
| 2435 |
+
"rewards/margins_max": 0.1240011602640152,
|
| 2436 |
+
"rewards/margins_min": 0.0478973314166069,
|
| 2437 |
+
"rewards/margins_std": 0.05381352826952934,
|
| 2438 |
+
"rewards/rejected": -0.017574548721313477,
|
| 2439 |
+
"step": 1340
|
| 2440 |
+
},
|
| 2441 |
+
{
|
| 2442 |
+
"epoch": 0.99,
|
| 2443 |
+
"grad_norm": 0.412109375,
|
| 2444 |
+
"learning_rate": 6.680708454906425e-11,
|
| 2445 |
+
"logits/chosen": -0.05887039750814438,
|
| 2446 |
+
"logits/rejected": 0.7168238759040833,
|
| 2447 |
+
"logps/chosen": -274.7958679199219,
|
| 2448 |
+
"logps/rejected": -234.88424682617188,
|
| 2449 |
+
"loss": 0.6506,
|
| 2450 |
+
"rewards/accuracies": 0.9750000238418579,
|
| 2451 |
+
"rewards/chosen": 0.06663821637630463,
|
| 2452 |
+
"rewards/margins": 0.08757266402244568,
|
| 2453 |
+
"rewards/margins_max": 0.1257934719324112,
|
| 2454 |
+
"rewards/margins_min": 0.049351852387189865,
|
| 2455 |
+
"rewards/margins_std": 0.05405237525701523,
|
| 2456 |
+
"rewards/rejected": -0.020934434607625008,
|
| 2457 |
+
"step": 1350
|
| 2458 |
+
},
|
| 2459 |
+
{
|
| 2460 |
+
"epoch": 1.0,
|
| 2461 |
+
"eval_logits/chosen": 0.744699239730835,
|
| 2462 |
+
"eval_logits/rejected": 0.8899587988853455,
|
| 2463 |
+
"eval_logps/chosen": -330.823974609375,
|
| 2464 |
+
"eval_logps/rejected": -316.1527404785156,
|
| 2465 |
+
"eval_loss": 0.6946464776992798,
|
| 2466 |
+
"eval_rewards/accuracies": 0.5195000171661377,
|
| 2467 |
+
"eval_rewards/chosen": 0.03155522421002388,
|
| 2468 |
+
"eval_rewards/margins": -0.0017884434200823307,
|
| 2469 |
+
"eval_rewards/margins_max": 0.09517286717891693,
|
| 2470 |
+
"eval_rewards/margins_min": -0.10408221185207367,
|
| 2471 |
+
"eval_rewards/margins_std": 0.06461790204048157,
|
| 2472 |
+
"eval_rewards/rejected": 0.03334366902709007,
|
| 2473 |
+
"eval_runtime": 837.0887,
|
| 2474 |
+
"eval_samples_per_second": 4.778,
|
| 2475 |
+
"eval_steps_per_second": 0.299,
|
| 2476 |
+
"step": 1359
|
| 2477 |
+
},
|
| 2478 |
+
{
|
| 2479 |
+
"epoch": 1.0,
|
| 2480 |
+
"step": 1359,
|
| 2481 |
+
"total_flos": 0.0,
|
| 2482 |
+
"train_loss": 0.6604200788417925,
|
| 2483 |
+
"train_runtime": 10358.1715,
|
| 2484 |
+
"train_samples_per_second": 2.099,
|
| 2485 |
+
"train_steps_per_second": 0.131
|
| 2486 |
+
}
|
| 2487 |
+
],
|
| 2488 |
+
"logging_steps": 10,
|
| 2489 |
+
"max_steps": 1359,
|
| 2490 |
+
"num_input_tokens_seen": 0,
|
| 2491 |
+
"num_train_epochs": 1,
|
| 2492 |
+
"save_steps": 100,
|
| 2493 |
+
"total_flos": 0.0,
|
| 2494 |
+
"train_batch_size": 2,
|
| 2495 |
+
"trial_name": null,
|
| 2496 |
+
"trial_params": null
|
| 2497 |
+
}
|