Model save

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+library_name: peft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+base_model: alignment-handbook/zephyr-7b-sft-full
+model-index:
+- name: nash_simple_online_iter_1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# nash_simple_online_iter_1
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6779
+- Rewards/chosen: 0.0306
+- Rewards/rejected: -0.0021
+- Rewards/accuracies: 0.6480
+- Rewards/margins: 0.0327
+- Logps/rejected: -257.7254
+- Logps/chosen: -280.9805
+- Logits/rejected: -2.6488
+- Logits/chosen: -2.7230
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 8
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6844        | 0.64  | 100  | 0.6779          | 0.0306         | -0.0021          | 0.6480             | 0.0327          | -257.7254      | -280.9805    | -2.6488         | -2.7230       |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6878455548714368,
+    "train_runtime": 6320.6894,
+    "train_samples": 20000,
+    "train_samples_per_second": 3.164,
+    "train_steps_per_second": 0.025
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6878455548714368,
+    "train_runtime": 6320.6894,
+    "train_samples": 20000,
+    "train_samples_per_second": 3.164,
+    "train_steps_per_second": 0.025
+}

trainer_state.json

@@ -0,0 +1,270 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9984,
+  "eval_steps": 100,
+  "global_step": 156,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.125e-07,
+      "logits/chosen": -2.783127546310425,
+      "logits/rejected": -2.8651208877563477,
+      "logps/chosen": -134.2432861328125,
+      "logps/rejected": -141.6387939453125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.125e-06,
+      "logits/chosen": -2.801163911819458,
+      "logits/rejected": -2.7942428588867188,
+      "logps/chosen": -112.86317443847656,
+      "logps/rejected": -110.90413665771484,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": 0.0007365171913988888,
+      "rewards/margins": -3.1015442800708115e-05,
+      "rewards/rejected": 0.0007675326196476817,
+      "step": 10
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.989935734988098e-06,
+      "logits/chosen": -2.7939412593841553,
+      "logits/rejected": -2.761488437652588,
+      "logps/chosen": -111.9773178100586,
+      "logps/rejected": -112.6773681640625,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": 0.007041769567877054,
+      "rewards/margins": 0.0005797661142423749,
+      "rewards/rejected": 0.006462003104388714,
+      "step": 20
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8776412907378845e-06,
+      "logits/chosen": -2.7452473640441895,
+      "logits/rejected": -2.71763277053833,
+      "logps/chosen": -110.77873229980469,
+      "logps/rejected": -116.26164245605469,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": 0.027953902259469032,
+      "rewards/margins": 0.002175498753786087,
+      "rewards/rejected": 0.025778403505682945,
+      "step": 30
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.646121984004666e-06,
+      "logits/chosen": -2.7981982231140137,
+      "logits/rejected": -2.760676860809326,
+      "logps/chosen": -110.84574890136719,
+      "logps/rejected": -111.5237808227539,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.05981600284576416,
+      "rewards/margins": 0.0045239622704684734,
+      "rewards/rejected": 0.055292047560214996,
+      "step": 40
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.3069871595684795e-06,
+      "logits/chosen": -2.7610268592834473,
+      "logits/rejected": -2.7524187564849854,
+      "logps/chosen": -109.18692779541016,
+      "logps/rejected": -110.45135498046875,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.07008253037929535,
+      "rewards/margins": 0.003140285611152649,
+      "rewards/rejected": 0.0669422522187233,
+      "step": 50
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.8772424536302565e-06,
+      "logits/chosen": -2.7594542503356934,
+      "logits/rejected": -2.7281312942504883,
+      "logps/chosen": -105.79144287109375,
+      "logps/rejected": -108.7798080444336,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.08012694865465164,
+      "rewards/margins": 0.008532116189599037,
+      "rewards/rejected": 0.07159483432769775,
+      "step": 60
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3784370602033572e-06,
+      "logits/chosen": -2.7187533378601074,
+      "logits/rejected": -2.6961326599121094,
+      "logps/chosen": -120.80352783203125,
+      "logps/rejected": -117.37217712402344,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.05428874492645264,
+      "rewards/margins": 0.007891224697232246,
+      "rewards/rejected": 0.04639752581715584,
+      "step": 70
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.835583164544139e-06,
+      "logits/chosen": -2.7139596939086914,
+      "logits/rejected": -2.6886355876922607,
+      "logps/chosen": -117.95060729980469,
+      "logps/rejected": -119.08524322509766,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": 0.04534279555082321,
+      "rewards/margins": 0.010814065113663673,
+      "rewards/rejected": 0.03452872857451439,
+      "step": 80
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2759017277414165e-06,
+      "logits/chosen": -2.7270092964172363,
+      "logits/rejected": -2.677152633666992,
+      "logps/chosen": -109.81172943115234,
+      "logps/rejected": -110.4306640625,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.026867974549531937,
+      "rewards/margins": 0.010544367134571075,
+      "rewards/rejected": 0.01632360741496086,
+      "step": 90
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7274575140626318e-06,
+      "logits/chosen": -2.720876932144165,
+      "logits/rejected": -2.7096195220947266,
+      "logps/chosen": -115.74837493896484,
+      "logps/rejected": -114.7901382446289,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": 0.042589299380779266,
+      "rewards/margins": 0.017415037378668785,
+      "rewards/rejected": 0.02517426386475563,
+      "step": 100
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/chosen": -2.722978353500366,
+      "eval_logits/rejected": -2.648787021636963,
+      "eval_logps/chosen": -280.98046875,
+      "eval_logps/rejected": -257.7253723144531,
+      "eval_loss": 0.677879810333252,
+      "eval_rewards/accuracies": 0.6480000019073486,
+      "eval_rewards/chosen": 0.03059276193380356,
+      "eval_rewards/margins": 0.032687921077013016,
+      "eval_rewards/rejected": -0.0020951556507498026,
+      "eval_runtime": 383.9393,
+      "eval_samples_per_second": 5.209,
+      "eval_steps_per_second": 0.651,
+      "step": 100
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.217751806485235e-06,
+      "logits/chosen": -2.6989266872406006,
+      "logits/rejected": -2.669929265975952,
+      "logps/chosen": -106.67137145996094,
+      "logps/rejected": -105.73466491699219,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": 0.05706842616200447,
+      "rewards/margins": 0.01945202611386776,
+      "rewards/rejected": 0.03761639818549156,
+      "step": 110
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.723433775328385e-07,
+      "logits/chosen": -2.6370861530303955,
+      "logits/rejected": -2.6202707290649414,
+      "logps/chosen": -107.36185455322266,
+      "logps/rejected": -108.76318359375,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": 0.06415753811597824,
+      "rewards/margins": 0.012921139597892761,
+      "rewards/rejected": 0.05123639851808548,
+      "step": 120
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.1356686569674344e-07,
+      "logits/chosen": -2.7071995735168457,
+      "logits/rejected": -2.6700949668884277,
+      "logps/chosen": -110.8349609375,
+      "logps/rejected": -116.52140045166016,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.062352847307920456,
+      "rewards/margins": 0.018708955496549606,
+      "rewards/rejected": 0.04364389181137085,
+      "step": 130
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.59412823400657e-07,
+      "logits/chosen": -2.7012362480163574,
+      "logits/rejected": -2.673867702484131,
+      "logps/chosen": -114.33103942871094,
+      "logps/rejected": -117.1007308959961,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.055680979043245316,
+      "rewards/margins": 0.020022045820951462,
+      "rewards/rejected": 0.03565892577171326,
+      "step": 140
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.262559558016325e-08,
+      "logits/chosen": -2.6588995456695557,
+      "logits/rejected": -2.643526077270508,
+      "logps/chosen": -107.59385681152344,
+      "logps/rejected": -118.29096984863281,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.05322548747062683,
+      "rewards/margins": 0.02210088260471821,
+      "rewards/rejected": 0.031124606728553772,
+      "step": 150
+    },
+    {
+      "epoch": 1.0,
+      "step": 156,
+      "total_flos": 0.0,
+      "train_loss": 0.6878455548714368,
+      "train_runtime": 6320.6894,
+      "train_samples_per_second": 3.164,
+      "train_steps_per_second": 0.025
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 156,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}