Model save

README.md

@@ -41,7 +41,6 @@ The following hyperparameters were used during training:
 - total_train_batch_size: 256
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- lr_scheduler_warmup_steps: 2
 - num_epochs: 4
 
 ### Training results

adapter_config.json

@@ -16,17 +16,17 @@
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 8,
+  "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
-    "k_proj",
-    "up_proj",
     "q_proj",
+    "up_proj",
     "v_proj",
     "down_proj",
-    "o_proj"
+    "gate_proj",
+    "o_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16b728db4f0ae6e7dc0c9b467dc3f7ce7140d6eff3e3ecd7466031c095b13d8d
-size 1080767344
+oid sha256:9bbc2b4236a166d28985440f9a5fb006ea8fecc10c1208ca13e4582cf325c565
+size 1103312056

trainer_state.json

@@ -10,67 +10,67 @@
   "log_history": [
     {
       "epoch": 0.4854368932038835,
-      "grad_norm": 1.100472092628479,
-      "learning_rate": 8.794946550048592e-05,
-      "loss": 0.9267,
+      "grad_norm": 0.6743159890174866,
+      "learning_rate": 8.786407766990292e-05,
+      "loss": 0.9212,
       "step": 250
     },
     {
       "epoch": 0.970873786407767,
-      "grad_norm": 0.9649342894554138,
-      "learning_rate": 7.580174927113704e-05,
-      "loss": 0.4675,
+      "grad_norm": 0.7125601768493652,
+      "learning_rate": 7.572815533980583e-05,
+      "loss": 0.4799,
       "step": 500
     },
     {
       "epoch": 1.4563106796116505,
-      "grad_norm": 0.7847822308540344,
-      "learning_rate": 6.365403304178815e-05,
-      "loss": 0.3979,
+      "grad_norm": 0.6055670380592346,
+      "learning_rate": 6.359223300970875e-05,
+      "loss": 0.4021,
       "step": 750
     },
     {
       "epoch": 1.941747572815534,
-      "grad_norm": 0.7165718674659729,
-      "learning_rate": 5.150631681243926e-05,
-      "loss": 0.3749,
+      "grad_norm": 0.5459219217300415,
+      "learning_rate": 5.145631067961165e-05,
+      "loss": 0.3765,
       "step": 1000
     },
     {
       "epoch": 2.4271844660194173,
-      "grad_norm": 0.6947001218795776,
-      "learning_rate": 3.9358600583090386e-05,
-      "loss": 0.3641,
+      "grad_norm": 0.5173175930976868,
+      "learning_rate": 3.9320388349514564e-05,
+      "loss": 0.365,
       "step": 1250
     },
     {
       "epoch": 2.912621359223301,
-      "grad_norm": 0.6668033599853516,
-      "learning_rate": 2.72108843537415e-05,
-      "loss": 0.3589,
+      "grad_norm": 0.5147077441215515,
+      "learning_rate": 2.7184466019417475e-05,
+      "loss": 0.3597,
       "step": 1500
     },
     {
       "epoch": 3.3980582524271843,
-      "grad_norm": 0.6627563238143921,
-      "learning_rate": 1.5063168124392615e-05,
-      "loss": 0.3522,
+      "grad_norm": 0.4768456220626831,
+      "learning_rate": 1.5048543689320387e-05,
+      "loss": 0.3529,
       "step": 1750
     },
     {
       "epoch": 3.883495145631068,
-      "grad_norm": 0.6149743795394897,
-      "learning_rate": 2.915451895043732e-06,
-      "loss": 0.349,
+      "grad_norm": 0.4576747715473175,
+      "learning_rate": 2.912621359223301e-06,
+      "loss": 0.3498,
       "step": 2000
     },
     {
       "epoch": 4.0,
       "step": 2060,
-      "total_flos": 4.996840775351132e+17,
-      "train_loss": 0.44594140932397935,
-      "train_runtime": 7769.6571,
-      "train_samples_per_second": 67.873,
+      "total_flos": 5.02561395295531e+17,
+      "train_loss": 0.44787726448577586,
+      "train_runtime": 7765.6442,
+      "train_samples_per_second": 67.908,
       "train_steps_per_second": 0.265
     }
   ],
@@ -91,7 +91,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.996840775351132e+17,
+  "total_flos": 5.02561395295531e+17,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:336c7dd55e42b14613ffcdd4b723c0dd1e62bb1d4d52835fe6220dbf73a73146
+oid sha256:006bef7a831664ec1478ce6887302f644555ecc4a4acaefb843072522b533855
 size 5240