Training in progress, step 500

.ipynb_checkpoints/lora_orpo-checkpoint.yaml

@@ -12,7 +12,7 @@ dataset: dpo_mix_en
 dataset_dir: data
 template: gemma
 cutoff_len: 1024
-max_samples: 1000
+# max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
 
@@ -28,8 +28,8 @@ push_to_hub: true
 hub_model_id: chchen/Gemma-7B-It-ORPO
 
 ### train
-per_device_train_batch_size: 1
-gradient_accumulation_steps: 16
+per_device_train_batch_size: 2
+gradient_accumulation_steps: 8
 learning_rate: 0.000005
 num_train_epochs: 3.0
 lr_scheduler_type: cosine
@@ -38,6 +38,6 @@ bf16: true
 
 ### eval
 val_size: 0.1
-per_device_eval_batch_size: 1
+per_device_eval_batch_size: 2
 evaluation_strategy: steps
 eval_steps: 500

.ipynb_checkpoints/tokenizer_config-checkpoint.json

@@ -0,0 +1,1759 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{{ '<bos>' + system_message }}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<start_of_turn>user\n' + content + '<end_of_turn>\n<start_of_turn>model\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<end_of_turn>\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

README.md

@@ -1,5 +1,5 @@
 ---
-license: gemma
+license: other
 library_name: peft
 tags:
 - llama-factory
@@ -9,28 +9,28 @@ tags:
 - generated_from_trainer
 base_model: google/gemma-7b-it
 model-index:
-- name: Gemma-7B-It-ORPO
+- name: orpo
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# Gemma-7B-It-ORPO
+# orpo
 
 This model is a fine-tuned version of [google/gemma-7b-it](https://huggingface.co/google/gemma-7b-it) on the dpo_mix_en dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.7794
-- Rewards/chosen: -0.1716
-- Rewards/rejected: -0.1920
-- Rewards/accuracies: 0.5600
-- Rewards/margins: 0.0204
-- Logps/rejected: -1.9200
-- Logps/chosen: -1.7164
-- Logits/rejected: 236.5044
-- Logits/chosen: 236.6770
-- Sft Loss: 1.7164
-- Odds Ratio Loss: 0.6304
+- Loss: 4.4269
+- Rewards/chosen: -0.4347
+- Rewards/rejected: -0.4158
+- Rewards/accuracies: 0.0
+- Rewards/margins: -0.0190
+- Logps/rejected: -4.1575
+- Logps/chosen: -4.3475
+- Logits/rejected: 198.3044
+- Logits/chosen: 195.7259
+- Sft Loss: 4.3475
+- Odds Ratio Loss: 0.7941
 
 ## Model description
 
@@ -50,15 +50,15 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-06
-- train_batch_size: 1
-- eval_batch_size: 1
+- train_batch_size: 2
+- eval_batch_size: 2
 - seed: 42
-- gradient_accumulation_steps: 16
+- gradient_accumulation_steps: 8
 - total_train_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 0.1
-- num_epochs: 3.0
+- num_epochs: 1.0
 
 ### Training results
 

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "k_proj",
     "v_proj",
+    "gate_proj",
+    "down_proj",
     "q_proj",
     "up_proj",
-    "gate_proj",
-    "o_proj",
-    "k_proj",
-    "down_proj"
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc4b6acb2d98e1fe0132e7b05219cd20ebed844e661990d0b1652210733570e4
+oid sha256:c53decec59c43bbf505c5ed52f992ac7218eeee5216ea6cc873f170589dadc41
 size 100059752

all_results.json

@@ -1,22 +1,22 @@
 {
-    "epoch": 2.986666666666667,
-    "eval_logits/chosen": 236.67703247070312,
-    "eval_logits/rejected": 236.50440979003906,
-    "eval_logps/chosen": -1.7163937091827393,
-    "eval_logps/rejected": -1.9199703931808472,
-    "eval_loss": 1.7794384956359863,
-    "eval_odds_ratio_loss": 0.6304484009742737,
-    "eval_rewards/accuracies": 0.5600000023841858,
-    "eval_rewards/chosen": -0.1716393679380417,
-    "eval_rewards/margins": 0.020357677713036537,
-    "eval_rewards/rejected": -0.19199703633785248,
-    "eval_runtime": 59.9808,
-    "eval_samples_per_second": 1.667,
-    "eval_sft_loss": 1.7163937091827393,
-    "eval_steps_per_second": 1.667,
-    "total_flos": 2.2023536924295168e+17,
-    "train_loss": 1.8747729460398357,
-    "train_runtime": 5642.3245,
-    "train_samples_per_second": 0.479,
-    "train_steps_per_second": 0.03
+    "epoch": 1.0,
+    "eval_logits/chosen": 195.72586059570312,
+    "eval_logits/rejected": 198.3043975830078,
+    "eval_logps/chosen": -4.347492694854736,
+    "eval_logps/rejected": -4.157506465911865,
+    "eval_loss": 4.426907539367676,
+    "eval_odds_ratio_loss": 0.79414963722229,
+    "eval_rewards/accuracies": 0.0,
+    "eval_rewards/chosen": -0.4347492754459381,
+    "eval_rewards/margins": -0.01899862289428711,
+    "eval_rewards/rejected": -0.415750652551651,
+    "eval_runtime": 0.2975,
+    "eval_samples_per_second": 3.361,
+    "eval_sft_loss": 4.347492694854736,
+    "eval_steps_per_second": 3.361,
+    "total_flos": 748013124648960.0,
+    "train_loss": 2.0597352981567383,
+    "train_runtime": 12.1534,
+    "train_samples_per_second": 0.741,
+    "train_steps_per_second": 0.082
 }

eval_results.json

@@ -1,17 +1,17 @@
 {
-    "epoch": 2.986666666666667,
-    "eval_logits/chosen": 236.67703247070312,
-    "eval_logits/rejected": 236.50440979003906,
-    "eval_logps/chosen": -1.7163937091827393,
-    "eval_logps/rejected": -1.9199703931808472,
-    "eval_loss": 1.7794384956359863,
-    "eval_odds_ratio_loss": 0.6304484009742737,
-    "eval_rewards/accuracies": 0.5600000023841858,
-    "eval_rewards/chosen": -0.1716393679380417,
-    "eval_rewards/margins": 0.020357677713036537,
-    "eval_rewards/rejected": -0.19199703633785248,
-    "eval_runtime": 59.9808,
-    "eval_samples_per_second": 1.667,
-    "eval_sft_loss": 1.7163937091827393,
-    "eval_steps_per_second": 1.667
+    "epoch": 1.0,
+    "eval_logits/chosen": 195.72586059570312,
+    "eval_logits/rejected": 198.3043975830078,
+    "eval_logps/chosen": -4.347492694854736,
+    "eval_logps/rejected": -4.157506465911865,
+    "eval_loss": 4.426907539367676,
+    "eval_odds_ratio_loss": 0.79414963722229,
+    "eval_rewards/accuracies": 0.0,
+    "eval_rewards/chosen": -0.4347492754459381,
+    "eval_rewards/margins": -0.01899862289428711,
+    "eval_rewards/rejected": -0.415750652551651,
+    "eval_runtime": 0.2975,
+    "eval_samples_per_second": 3.361,
+    "eval_sft_loss": 4.347492694854736,
+    "eval_steps_per_second": 3.361
 }

lora_orpo.yaml

@@ -12,7 +12,7 @@ dataset: dpo_mix_en
 dataset_dir: data
 template: gemma
 cutoff_len: 1024
-max_samples: 1000
+# max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
 
@@ -28,8 +28,8 @@ push_to_hub: true
 hub_model_id: chchen/Gemma-7B-It-ORPO
 
 ### train
-per_device_train_batch_size: 1
-gradient_accumulation_steps: 16
+per_device_train_batch_size: 2
+gradient_accumulation_steps: 8
 learning_rate: 0.000005
 num_train_epochs: 3.0
 lr_scheduler_type: cosine
@@ -38,6 +38,6 @@ bf16: true
 
 ### eval
 val_size: 0.1
-per_device_eval_batch_size: 1
+per_device_eval_batch_size: 2
 evaluation_strategy: steps
 eval_steps: 500

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.986666666666667,
-    "total_flos": 2.2023536924295168e+17,
-    "train_loss": 1.8747729460398357,
-    "train_runtime": 5642.3245,
-    "train_samples_per_second": 0.479,
-    "train_steps_per_second": 0.03
+    "epoch": 1.0,
+    "total_flos": 748013124648960.0,
+    "train_loss": 2.0597352981567383,
+    "train_runtime": 12.1534,
+    "train_samples_per_second": 0.741,
+    "train_steps_per_second": 0.082
 }

trainer_log.jsonl

@@ -1,17 +1,51 @@
-{"current_steps": 10, "total_steps": 168, "loss": 2.1168, "accuracy": 0.5249999761581421, "learning_rate": 4.957230266673969e-06, "epoch": 0.17777777777777778, "percentage": 5.95, "elapsed_time": "0:05:38", "remaining_time": "1:29:04"}
-{"current_steps": 20, "total_steps": 168, "loss": 2.1946, "accuracy": 0.543749988079071, "learning_rate": 4.828686741593921e-06, "epoch": 0.35555555555555557, "percentage": 11.9, "elapsed_time": "0:11:08", "remaining_time": "1:22:23"}
-{"current_steps": 30, "total_steps": 168, "loss": 2.0246, "accuracy": 0.53125, "learning_rate": 4.618852307232078e-06, "epoch": 0.5333333333333333, "percentage": 17.86, "elapsed_time": "0:16:57", "remaining_time": "1:18:02"}
-{"current_steps": 40, "total_steps": 168, "loss": 1.913, "accuracy": 0.5874999761581421, "learning_rate": 4.335051964269395e-06, "epoch": 0.7111111111111111, "percentage": 23.81, "elapsed_time": "0:22:22", "remaining_time": "1:11:37"}
-{"current_steps": 50, "total_steps": 168, "loss": 2.12, "accuracy": 0.5375000238418579, "learning_rate": 3.987192750660719e-06, "epoch": 0.8888888888888888, "percentage": 29.76, "elapsed_time": "0:27:59", "remaining_time": "1:06:03"}
-{"current_steps": 60, "total_steps": 168, "loss": 1.9418, "accuracy": 0.5625, "learning_rate": 3.587417902020876e-06, "epoch": 1.0666666666666667, "percentage": 35.71, "elapsed_time": "0:33:36", "remaining_time": "1:00:30"}
-{"current_steps": 70, "total_steps": 168, "loss": 1.8513, "accuracy": 0.5625, "learning_rate": 3.1496829497545268e-06, "epoch": 1.2444444444444445, "percentage": 41.67, "elapsed_time": "0:39:07", "remaining_time": "0:54:46"}
-{"current_steps": 80, "total_steps": 168, "loss": 1.8758, "accuracy": 0.574999988079071, "learning_rate": 2.6892685546987724e-06, "epoch": 1.4222222222222223, "percentage": 47.62, "elapsed_time": "0:44:51", "remaining_time": "0:49:20"}
-{"current_steps": 90, "total_steps": 168, "loss": 1.7086, "accuracy": 0.543749988079071, "learning_rate": 2.2222470825144806e-06, "epoch": 1.6, "percentage": 53.57, "elapsed_time": "0:50:22", "remaining_time": "0:43:39"}
-{"current_steps": 100, "total_steps": 168, "loss": 1.8133, "accuracy": 0.5625, "learning_rate": 1.7649215418673847e-06, "epoch": 1.7777777777777777, "percentage": 59.52, "elapsed_time": "0:55:57", "remaining_time": "0:38:03"}
-{"current_steps": 110, "total_steps": 168, "loss": 1.7876, "accuracy": 0.518750011920929, "learning_rate": 1.3332564712129845e-06, "epoch": 1.9555555555555557, "percentage": 65.48, "elapsed_time": "1:01:36", "remaining_time": "0:32:28"}
-{"current_steps": 120, "total_steps": 168, "loss": 1.7863, "accuracy": 0.53125, "learning_rate": 9.423206410612498e-07, "epoch": 2.1333333333333333, "percentage": 71.43, "elapsed_time": "1:07:20", "remaining_time": "0:26:56"}
-{"current_steps": 130, "total_steps": 168, "loss": 1.692, "accuracy": 0.5625, "learning_rate": 6.057610261367044e-07, "epoch": 2.311111111111111, "percentage": 77.38, "elapsed_time": "1:13:01", "remaining_time": "0:21:20"}
-{"current_steps": 140, "total_steps": 168, "loss": 1.6376, "accuracy": 0.606249988079071, "learning_rate": 3.3532641026504415e-07, "epoch": 2.488888888888889, "percentage": 83.33, "elapsed_time": "1:18:21", "remaining_time": "0:15:40"}
-{"current_steps": 150, "total_steps": 168, "loss": 1.8891, "accuracy": 0.574999988079071, "learning_rate": 1.4045725421448332e-07, "epoch": 2.6666666666666665, "percentage": 89.29, "elapsed_time": "1:24:09", "remaining_time": "0:10:05"}
-{"current_steps": 160, "total_steps": 168, "loss": 1.6214, "accuracy": 0.6000000238418579, "learning_rate": 2.7956143581177874e-08, "epoch": 2.8444444444444446, "percentage": 95.24, "elapsed_time": "1:29:35", "remaining_time": "0:04:28"}
-{"current_steps": 168, "total_steps": 168, "epoch": 2.986666666666667, "percentage": 100.0, "elapsed_time": "1:34:02", "remaining_time": "0:00:00"}
+{"current_steps": 10, "total_steps": 1686, "loss": 2.2441, "accuracy": 0.4375, "learning_rate": 4.9995745934141085e-06, "epoch": 0.017781729273171815, "percentage": 0.59, "elapsed_time": "0:02:09", "remaining_time": "6:02:30"}
+{"current_steps": 20, "total_steps": 1686, "loss": 2.4841, "accuracy": 0.42500001192092896, "learning_rate": 4.9982812903243405e-06, "epoch": 0.03556345854634363, "percentage": 1.19, "elapsed_time": "0:04:20", "remaining_time": "6:01:43"}
+{"current_steps": 30, "total_steps": 1686, "loss": 2.4103, "accuracy": 0.5625, "learning_rate": 4.996120496405222e-06, "epoch": 0.05334518781951545, "percentage": 1.78, "elapsed_time": "0:06:30", "remaining_time": "5:59:09"}
+{"current_steps": 40, "total_steps": 1686, "loss": 2.2037, "accuracy": 0.5625, "learning_rate": 4.99309296196014e-06, "epoch": 0.07112691709268726, "percentage": 2.37, "elapsed_time": "0:08:38", "remaining_time": "5:55:20"}
+{"current_steps": 50, "total_steps": 1686, "loss": 2.2102, "accuracy": 0.550000011920929, "learning_rate": 4.989199738255166e-06, "epoch": 0.08890864636585907, "percentage": 2.97, "elapsed_time": "0:10:47", "remaining_time": "5:53:20"}
+{"current_steps": 60, "total_steps": 1686, "loss": 2.2519, "accuracy": 0.5249999761581421, "learning_rate": 4.984442177154031e-06, "epoch": 0.1066903756390309, "percentage": 3.56, "elapsed_time": "0:13:01", "remaining_time": "5:52:45"}
+{"current_steps": 70, "total_steps": 1686, "loss": 2.2226, "accuracy": 0.4749999940395355, "learning_rate": 4.978821930648704e-06, "epoch": 0.12447210491220272, "percentage": 4.15, "elapsed_time": "0:15:14", "remaining_time": "5:51:52"}
+{"current_steps": 80, "total_steps": 1686, "loss": 2.0967, "accuracy": 0.48750001192092896, "learning_rate": 4.97234095028576e-06, "epoch": 0.14225383418537452, "percentage": 4.74, "elapsed_time": "0:17:31", "remaining_time": "5:51:51"}
+{"current_steps": 90, "total_steps": 1686, "loss": 1.8703, "accuracy": 0.512499988079071, "learning_rate": 4.965001486488743e-06, "epoch": 0.16003556345854633, "percentage": 5.34, "elapsed_time": "0:19:42", "remaining_time": "5:49:33"}
+{"current_steps": 100, "total_steps": 1686, "loss": 1.8365, "accuracy": 0.59375, "learning_rate": 4.956806087776732e-06, "epoch": 0.17781729273171815, "percentage": 5.93, "elapsed_time": "0:21:52", "remaining_time": "5:46:59"}
+{"current_steps": 110, "total_steps": 1686, "loss": 1.7686, "accuracy": 0.581250011920929, "learning_rate": 4.947757599879411e-06, "epoch": 0.19559902200489, "percentage": 6.52, "elapsed_time": "0:23:55", "remaining_time": "5:42:48"}
+{"current_steps": 120, "total_steps": 1686, "loss": 1.6182, "accuracy": 0.5, "learning_rate": 4.937859164748931e-06, "epoch": 0.2133807512780618, "percentage": 7.12, "elapsed_time": "0:26:00", "remaining_time": "5:39:22"}
+{"current_steps": 130, "total_steps": 1686, "loss": 1.6813, "accuracy": 0.53125, "learning_rate": 4.92711421946891e-06, "epoch": 0.23116248055123362, "percentage": 7.71, "elapsed_time": "0:28:18", "remaining_time": "5:38:50"}
+{"current_steps": 140, "total_steps": 1686, "loss": 1.5772, "accuracy": 0.5874999761581421, "learning_rate": 4.915526495060961e-06, "epoch": 0.24894420982440543, "percentage": 8.3, "elapsed_time": "0:30:33", "remaining_time": "5:37:25"}
+{"current_steps": 150, "total_steps": 1686, "loss": 1.6165, "accuracy": 0.5062500238418579, "learning_rate": 4.903100015189153e-06, "epoch": 0.26672593909757725, "percentage": 8.9, "elapsed_time": "0:32:45", "remaining_time": "5:35:27"}
+{"current_steps": 160, "total_steps": 1686, "loss": 1.6882, "accuracy": 0.543749988079071, "learning_rate": 4.889839094762848e-06, "epoch": 0.28450766837074903, "percentage": 9.49, "elapsed_time": "0:34:52", "remaining_time": "5:32:37"}
+{"current_steps": 170, "total_steps": 1686, "loss": 1.5978, "accuracy": 0.5249999761581421, "learning_rate": 4.875748338438416e-06, "epoch": 0.3022893976439209, "percentage": 10.08, "elapsed_time": "0:36:58", "remaining_time": "5:29:42"}
+{"current_steps": 180, "total_steps": 1686, "loss": 1.5646, "accuracy": 0.612500011920929, "learning_rate": 4.8608326390203386e-06, "epoch": 0.32007112691709266, "percentage": 10.68, "elapsed_time": "0:39:09", "remaining_time": "5:27:40"}
+{"current_steps": 190, "total_steps": 1686, "loss": 1.5391, "accuracy": 0.4937500059604645, "learning_rate": 4.845097175762251e-06, "epoch": 0.3378528561902645, "percentage": 11.27, "elapsed_time": "0:41:20", "remaining_time": "5:25:29"}
+{"current_steps": 200, "total_steps": 1686, "loss": 1.5928, "accuracy": 0.48750001192092896, "learning_rate": 4.8285474125685286e-06, "epoch": 0.3556345854634363, "percentage": 11.86, "elapsed_time": "0:43:28", "remaining_time": "5:23:01"}
+{"current_steps": 210, "total_steps": 1686, "loss": 1.5627, "accuracy": 0.574999988079071, "learning_rate": 4.811189096097025e-06, "epoch": 0.37341631473660813, "percentage": 12.46, "elapsed_time": "0:45:50", "remaining_time": "5:22:14"}
+{"current_steps": 220, "total_steps": 1686, "loss": 1.5539, "accuracy": 0.512499988079071, "learning_rate": 4.793028253763633e-06, "epoch": 0.39119804400978, "percentage": 13.05, "elapsed_time": "0:48:03", "remaining_time": "5:20:11"}
+{"current_steps": 230, "total_steps": 1686, "loss": 1.421, "accuracy": 0.59375, "learning_rate": 4.774071191649352e-06, "epoch": 0.40897977328295176, "percentage": 13.64, "elapsed_time": "0:50:17", "remaining_time": "5:18:21"}
+{"current_steps": 240, "total_steps": 1686, "loss": 1.5553, "accuracy": 0.4749999940395355, "learning_rate": 4.7543244923105975e-06, "epoch": 0.4267615025561236, "percentage": 14.23, "elapsed_time": "0:52:30", "remaining_time": "5:16:22"}
+{"current_steps": 250, "total_steps": 1686, "loss": 1.6181, "accuracy": 0.53125, "learning_rate": 4.733795012493506e-06, "epoch": 0.4445432318292954, "percentage": 14.83, "elapsed_time": "0:54:47", "remaining_time": "5:14:40"}
+{"current_steps": 260, "total_steps": 1686, "loss": 1.3796, "accuracy": 0.5874999761581421, "learning_rate": 4.712489880753035e-06, "epoch": 0.46232496110246724, "percentage": 15.42, "elapsed_time": "0:57:02", "remaining_time": "5:12:50"}
+{"current_steps": 270, "total_steps": 1686, "loss": 1.5056, "accuracy": 0.59375, "learning_rate": 4.690416494977673e-06, "epoch": 0.480106690375639, "percentage": 16.01, "elapsed_time": "0:59:15", "remaining_time": "5:10:48"}
+{"current_steps": 280, "total_steps": 1686, "loss": 1.5385, "accuracy": 0.5625, "learning_rate": 4.667582519820639e-06, "epoch": 0.49788841964881086, "percentage": 16.61, "elapsed_time": "1:01:38", "remaining_time": "5:09:29"}
+{"current_steps": 290, "total_steps": 1686, "loss": 1.4582, "accuracy": 0.5687500238418579, "learning_rate": 4.643995884038443e-06, "epoch": 0.5156701489219827, "percentage": 17.2, "elapsed_time": "1:03:51", "remaining_time": "5:07:24"}
+{"current_steps": 300, "total_steps": 1686, "loss": 1.4621, "accuracy": 0.518750011920929, "learning_rate": 4.6196647777377475e-06, "epoch": 0.5334518781951545, "percentage": 17.79, "elapsed_time": "1:06:07", "remaining_time": "5:05:29"}
+{"current_steps": 310, "total_steps": 1686, "loss": 1.5181, "accuracy": 0.512499988079071, "learning_rate": 4.59459764953147e-06, "epoch": 0.5512336074683263, "percentage": 18.39, "elapsed_time": "1:08:29", "remaining_time": "5:03:59"}
+{"current_steps": 320, "total_steps": 1686, "loss": 1.4444, "accuracy": 0.606249988079071, "learning_rate": 4.568803203605133e-06, "epoch": 0.5690153367414981, "percentage": 18.98, "elapsed_time": "1:10:45", "remaining_time": "5:02:01"}
+{"current_steps": 330, "total_steps": 1686, "loss": 1.4861, "accuracy": 0.4937500059604645, "learning_rate": 4.542290396694462e-06, "epoch": 0.58679706601467, "percentage": 19.57, "elapsed_time": "1:13:04", "remaining_time": "5:00:16"}
+{"current_steps": 340, "total_steps": 1686, "loss": 1.4802, "accuracy": 0.550000011920929, "learning_rate": 4.515068434975298e-06, "epoch": 0.6045787952878418, "percentage": 20.17, "elapsed_time": "1:15:14", "remaining_time": "4:57:53"}
+{"current_steps": 350, "total_steps": 1686, "loss": 1.4475, "accuracy": 0.4937500059604645, "learning_rate": 4.487146770866887e-06, "epoch": 0.6223605245610135, "percentage": 20.76, "elapsed_time": "1:17:30", "remaining_time": "4:55:50"}
+{"current_steps": 360, "total_steps": 1686, "loss": 1.5213, "accuracy": 0.53125, "learning_rate": 4.458535099749666e-06, "epoch": 0.6401422538341853, "percentage": 21.35, "elapsed_time": "1:19:40", "remaining_time": "4:53:28"}
+{"current_steps": 370, "total_steps": 1686, "loss": 1.504, "accuracy": 0.5562499761581421, "learning_rate": 4.429243356598694e-06, "epoch": 0.6579239831073572, "percentage": 21.95, "elapsed_time": "1:21:52", "remaining_time": "4:51:11"}
+{"current_steps": 380, "total_steps": 1686, "loss": 1.4575, "accuracy": 0.5249999761581421, "learning_rate": 4.399281712533875e-06, "epoch": 0.675705712380529, "percentage": 22.54, "elapsed_time": "1:23:56", "remaining_time": "4:48:30"}
+{"current_steps": 390, "total_steps": 1686, "loss": 1.404, "accuracy": 0.5062500238418579, "learning_rate": 4.368660571288192e-06, "epoch": 0.6934874416537008, "percentage": 23.13, "elapsed_time": "1:26:17", "remaining_time": "4:46:43"}
+{"current_steps": 400, "total_steps": 1686, "loss": 1.5334, "accuracy": 0.518750011920929, "learning_rate": 4.337390565595163e-06, "epoch": 0.7112691709268726, "percentage": 23.72, "elapsed_time": "1:28:31", "remaining_time": "4:44:37"}
+{"current_steps": 410, "total_steps": 1686, "loss": 1.4299, "accuracy": 0.59375, "learning_rate": 4.305482553496786e-06, "epoch": 0.7290509002000445, "percentage": 24.32, "elapsed_time": "1:30:40", "remaining_time": "4:42:13"}
+{"current_steps": 420, "total_steps": 1686, "loss": 1.4431, "accuracy": 0.574999988079071, "learning_rate": 4.272947614573244e-06, "epoch": 0.7468326294732163, "percentage": 24.91, "elapsed_time": "1:32:50", "remaining_time": "4:39:50"}
+{"current_steps": 430, "total_steps": 1686, "loss": 1.43, "accuracy": 0.550000011920929, "learning_rate": 4.23979704609569e-06, "epoch": 0.7646143587463881, "percentage": 25.5, "elapsed_time": "1:35:02", "remaining_time": "4:37:37"}
+{"current_steps": 440, "total_steps": 1686, "loss": 1.5216, "accuracy": 0.59375, "learning_rate": 4.206042359103435e-06, "epoch": 0.78239608801956, "percentage": 26.1, "elapsed_time": "1:37:15", "remaining_time": "4:35:24"}
+{"current_steps": 450, "total_steps": 1686, "loss": 1.4601, "accuracy": 0.5249999761581421, "learning_rate": 4.17169527440691e-06, "epoch": 0.8001778172927317, "percentage": 26.69, "elapsed_time": "1:39:21", "remaining_time": "4:32:53"}
+{"current_steps": 460, "total_steps": 1686, "loss": 1.3289, "accuracy": 0.543749988079071, "learning_rate": 4.136767718517797e-06, "epoch": 0.8179595465659035, "percentage": 27.28, "elapsed_time": "1:41:35", "remaining_time": "4:30:44"}
+{"current_steps": 470, "total_steps": 1686, "loss": 1.4238, "accuracy": 0.4625000059604645, "learning_rate": 4.1012718195077196e-06, "epoch": 0.8357412758390753, "percentage": 27.88, "elapsed_time": "1:43:49", "remaining_time": "4:28:37"}
+{"current_steps": 480, "total_steps": 1686, "loss": 1.3953, "accuracy": 0.574999988079071, "learning_rate": 4.065219902796953e-06, "epoch": 0.8535230051122472, "percentage": 28.47, "elapsed_time": "1:45:55", "remaining_time": "4:26:07"}
+{"current_steps": 490, "total_steps": 1686, "loss": 1.4069, "accuracy": 0.612500011920929, "learning_rate": 4.028624486874608e-06, "epoch": 0.871304734385419, "percentage": 29.06, "elapsed_time": "1:48:07", "remaining_time": "4:23:55"}
+{"current_steps": 500, "total_steps": 1686, "loss": 1.5041, "accuracy": 0.5249999761581421, "learning_rate": 3.99149827895177e-06, "epoch": 0.8890864636585908, "percentage": 29.66, "elapsed_time": "1:50:26", "remaining_time": "4:21:57"}
+{"current_steps": 500, "total_steps": 1686, "eval_loss": 1.418463945388794, "epoch": 0.8890864636585908, "percentage": 29.66, "elapsed_time": "1:54:03", "remaining_time": "4:30:33"}

trainer_state.json

@@ -1,302 +1,30 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.986666666666667,
-  "eval_steps": 500,
-  "global_step": 168,
+  "epoch": 1.0,
+  "eval_steps": 10,
+  "global_step": 1,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.17777777777777778,
-      "grad_norm": 1.983818769454956,
-      "learning_rate": 4.957230266673969e-06,
-      "logits/chosen": 217.77822875976562,
-      "logits/rejected": 217.4987335205078,
-      "logps/chosen": -2.053837299346924,
-      "logps/rejected": -2.262728691101074,
-      "loss": 2.1168,
-      "odds_ratio_loss": 0.6295818090438843,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.20538373291492462,
-      "rewards/margins": 0.020889144390821457,
-      "rewards/rejected": -0.22627286612987518,
-      "sft_loss": 2.053837299346924,
-      "step": 10
-    },
-    {
-      "epoch": 0.35555555555555557,
-      "grad_norm": 1.9484807252883911,
-      "learning_rate": 4.828686741593921e-06,
-      "logits/chosen": 219.17361450195312,
-      "logits/rejected": 218.8988037109375,
-      "logps/chosen": -2.1305088996887207,
-      "logps/rejected": -2.402930736541748,
-      "loss": 2.1946,
-      "odds_ratio_loss": 0.6408200263977051,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.21305091679096222,
-      "rewards/margins": 0.027242189273238182,
-      "rewards/rejected": -0.24029311537742615,
-      "sft_loss": 2.1305088996887207,
-      "step": 20
-    },
-    {
-      "epoch": 0.5333333333333333,
-      "grad_norm": 1.7125110626220703,
-      "learning_rate": 4.618852307232078e-06,
-      "logits/chosen": 221.2677459716797,
-      "logits/rejected": 221.5433807373047,
-      "logps/chosen": -1.9636151790618896,
-      "logps/rejected": -2.2476425170898438,
-      "loss": 2.0246,
-      "odds_ratio_loss": 0.6097511053085327,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.19636152684688568,
-      "rewards/margins": 0.028402745723724365,
-      "rewards/rejected": -0.22476427257061005,
-      "sft_loss": 1.9636151790618896,
-      "step": 30
-    },
-    {
-      "epoch": 0.7111111111111111,
-      "grad_norm": 3.0041399002075195,
-      "learning_rate": 4.335051964269395e-06,
-      "logits/chosen": 217.79830932617188,
-      "logits/rejected": 218.62911987304688,
-      "logps/chosen": -1.8520755767822266,
-      "logps/rejected": -2.1365625858306885,
-      "loss": 1.913,
-      "odds_ratio_loss": 0.6088087558746338,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.1852075308561325,
-      "rewards/margins": 0.028448715806007385,
-      "rewards/rejected": -0.2136562615633011,
-      "sft_loss": 1.8520755767822266,
-      "step": 40
-    },
-    {
-      "epoch": 0.8888888888888888,
-      "grad_norm": 3.342698335647583,
-      "learning_rate": 3.987192750660719e-06,
-      "logits/chosen": 225.99374389648438,
-      "logits/rejected": 225.8725128173828,
-      "logps/chosen": -2.055860996246338,
-      "logps/rejected": -2.403578996658325,
-      "loss": 2.12,
-      "odds_ratio_loss": 0.6415398716926575,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.20558610558509827,
-      "rewards/margins": 0.03477178141474724,
-      "rewards/rejected": -0.240357905626297,
-      "sft_loss": 2.055860996246338,
-      "step": 50
-    },
-    {
-      "epoch": 1.0666666666666667,
-      "grad_norm": 3.5346407890319824,
-      "learning_rate": 3.587417902020876e-06,
-      "logits/chosen": 227.7618408203125,
-      "logits/rejected": 229.3314971923828,
-      "logps/chosen": -1.8825321197509766,
-      "logps/rejected": -2.1969597339630127,
-      "loss": 1.9418,
-      "odds_ratio_loss": 0.5927931070327759,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.1882532387971878,
-      "rewards/margins": 0.03144273906946182,
-      "rewards/rejected": -0.21969597041606903,
-      "sft_loss": 1.8825321197509766,
-      "step": 60
-    },
-    {
-      "epoch": 1.2444444444444445,
-      "grad_norm": 2.7227234840393066,
-      "learning_rate": 3.1496829497545268e-06,
-      "logits/chosen": 228.9124298095703,
-      "logits/rejected": 228.7540740966797,
-      "logps/chosen": -1.7887885570526123,
-      "logps/rejected": -2.0250189304351807,
-      "loss": 1.8513,
-      "odds_ratio_loss": 0.6251059770584106,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.17887887358665466,
-      "rewards/margins": 0.023623019456863403,
-      "rewards/rejected": -0.20250189304351807,
-      "sft_loss": 1.7887885570526123,
-      "step": 70
-    },
-    {
-      "epoch": 1.4222222222222223,
-      "grad_norm": 1.4925893545150757,
-      "learning_rate": 2.6892685546987724e-06,
-      "logits/chosen": 233.6576385498047,
-      "logits/rejected": 233.0616455078125,
-      "logps/chosen": -1.8144474029541016,
-      "logps/rejected": -2.061004638671875,
-      "loss": 1.8758,
-      "odds_ratio_loss": 0.6132601499557495,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.18144474923610687,
-      "rewards/margins": 0.024655740708112717,
-      "rewards/rejected": -0.2061004638671875,
-      "sft_loss": 1.8144474029541016,
-      "step": 80
-    },
-    {
-      "epoch": 1.6,
-      "grad_norm": 1.5339916944503784,
-      "learning_rate": 2.2222470825144806e-06,
-      "logits/chosen": 231.31655883789062,
-      "logits/rejected": 231.7608184814453,
-      "logps/chosen": -1.6470849514007568,
-      "logps/rejected": -1.9702856540679932,
-      "loss": 1.7086,
-      "odds_ratio_loss": 0.6146546602249146,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.16470849514007568,
-      "rewards/margins": 0.0323200486600399,
-      "rewards/rejected": -0.19702854752540588,
-      "sft_loss": 1.6470849514007568,
-      "step": 90
-    },
-    {
-      "epoch": 1.7777777777777777,
-      "grad_norm": 1.0663031339645386,
-      "learning_rate": 1.7649215418673847e-06,
-      "logits/chosen": 235.10598754882812,
-      "logits/rejected": 235.50381469726562,
-      "logps/chosen": -1.750628113746643,
-      "logps/rejected": -1.9882148504257202,
-      "loss": 1.8133,
-      "odds_ratio_loss": 0.626555323600769,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.17506280541419983,
-      "rewards/margins": 0.023758674040436745,
-      "rewards/rejected": -0.19882148504257202,
-      "sft_loss": 1.750628113746643,
-      "step": 100
-    },
-    {
-      "epoch": 1.9555555555555557,
-      "grad_norm": 1.1423336267471313,
-      "learning_rate": 1.3332564712129845e-06,
-      "logits/chosen": 236.2017059326172,
-      "logits/rejected": 236.12319946289062,
-      "logps/chosen": -1.7247778177261353,
-      "logps/rejected": -1.9412933588027954,
-      "loss": 1.7876,
-      "odds_ratio_loss": 0.6279042363166809,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -0.17247776687145233,
-      "rewards/margins": 0.02165156602859497,
-      "rewards/rejected": -0.1941293478012085,
-      "sft_loss": 1.7247778177261353,
-      "step": 110
-    },
-    {
-      "epoch": 2.1333333333333333,
-      "grad_norm": 2.996070623397827,
-      "learning_rate": 9.423206410612498e-07,
-      "logits/chosen": 233.90518188476562,
-      "logits/rejected": 234.82778930664062,
-      "logps/chosen": -1.7220882177352905,
-      "logps/rejected": -1.962699294090271,
-      "loss": 1.7863,
-      "odds_ratio_loss": 0.6424781084060669,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.17220883071422577,
-      "rewards/margins": 0.024061108008027077,
-      "rewards/rejected": -0.1962699443101883,
-      "sft_loss": 1.7220882177352905,
-      "step": 120
-    },
-    {
-      "epoch": 2.311111111111111,
-      "grad_norm": 0.8977892994880676,
-      "learning_rate": 6.057610261367044e-07,
-      "logits/chosen": 233.8168487548828,
-      "logits/rejected": 233.524658203125,
-      "logps/chosen": -1.6289135217666626,
-      "logps/rejected": -1.8276309967041016,
-      "loss": 1.692,
-      "odds_ratio_loss": 0.630408525466919,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.16289135813713074,
-      "rewards/margins": 0.019871745258569717,
-      "rewards/rejected": -0.18276306986808777,
-      "sft_loss": 1.6289135217666626,
-      "step": 130
-    },
-    {
-      "epoch": 2.488888888888889,
-      "grad_norm": 1.3126909732818604,
-      "learning_rate": 3.3532641026504415e-07,
-      "logits/chosen": 237.79580688476562,
-      "logits/rejected": 237.5025634765625,
-      "logps/chosen": -1.577689528465271,
-      "logps/rejected": -1.9474560022354126,
-      "loss": 1.6376,
-      "odds_ratio_loss": 0.598879337310791,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.15776896476745605,
-      "rewards/margins": 0.0369766540825367,
-      "rewards/rejected": -0.19474558532238007,
-      "sft_loss": 1.577689528465271,
-      "step": 140
-    },
-    {
-      "epoch": 2.6666666666666665,
-      "grad_norm": 2.6702768802642822,
-      "learning_rate": 1.4045725421448332e-07,
-      "logits/chosen": 238.22622680664062,
-      "logits/rejected": 238.50765991210938,
-      "logps/chosen": -1.829874038696289,
-      "logps/rejected": -2.089751958847046,
-      "loss": 1.8891,
-      "odds_ratio_loss": 0.5918877720832825,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.18298740684986115,
-      "rewards/margins": 0.02598779834806919,
-      "rewards/rejected": -0.2089751958847046,
-      "sft_loss": 1.829874038696289,
-      "step": 150
-    },
-    {
-      "epoch": 2.8444444444444446,
-      "grad_norm": 1.5213804244995117,
-      "learning_rate": 2.7956143581177874e-08,
-      "logits/chosen": 237.4580078125,
-      "logits/rejected": 237.25192260742188,
-      "logps/chosen": -1.562623143196106,
-      "logps/rejected": -1.8317865133285522,
-      "loss": 1.6214,
-      "odds_ratio_loss": 0.587682843208313,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.15626230835914612,
-      "rewards/margins": 0.026916349306702614,
-      "rewards/rejected": -0.18317866325378418,
-      "sft_loss": 1.562623143196106,
-      "step": 160
-    },
-    {
-      "epoch": 2.986666666666667,
-      "step": 168,
-      "total_flos": 2.2023536924295168e+17,
-      "train_loss": 1.8747729460398357,
-      "train_runtime": 5642.3245,
-      "train_samples_per_second": 0.479,
-      "train_steps_per_second": 0.03
+      "epoch": 1.0,
+      "step": 1,
+      "total_flos": 748013124648960.0,
+      "train_loss": 2.0597352981567383,
+      "train_runtime": 12.1534,
+      "train_samples_per_second": 0.741,
+      "train_steps_per_second": 0.082
     }
   ],
   "logging_steps": 10,
-  "max_steps": 168,
+  "max_steps": 1,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
-  "save_steps": 500,
-  "total_flos": 2.2023536924295168e+17,
-  "train_batch_size": 1,
+  "num_train_epochs": 1,
+  "save_steps": 10,
+  "total_flos": 748013124648960.0,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f843c7a8a89253f57fbbf07b31914c7257caaec1c6f05fbf07eff4da859d27c
+oid sha256:cdc5ace08fcc677baafd248b5cbfffe7c70bc8b231764bd01d9edd7a5a5a822d
 size 5176