Model save

README.md

@@ -18,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0922
+- Loss: 11.6801
 
 ## Model description
 
@@ -54,7 +54,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.3681        | 1.0   | 272  | 1.0922          |
+| 27787.675     | 1.0   | 272  | 11.6801         |
 
 
 ### Framework versions

adapter_config.json

@@ -17,9 +17,9 @@
   "revision": null,
   "target_modules": [
     "q_proj",
+    "k_proj",
     "v_proj",
-    "o_proj",
-    "k_proj"
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:46933e34b2e227ff195e628e2c8b61bf212c9ef7309bda18a4978b0225e175c2
+oid sha256:5f5e96ad2271a638d3a39cdfed716fd42448e7931922a043f53eae00d68e04d6
 size 109086672

all_results.json

@@ -1,13 +1,13 @@
 {
     "epoch": 1.0,
-    "eval_loss": 1.0922234058380127,
-    "eval_runtime": 2385.5237,
+    "eval_loss": 11.680082321166992,
+    "eval_runtime": 2370.9651,
     "eval_samples": 23110,
-    "eval_samples_per_second": 6.469,
-    "eval_steps_per_second": 0.101,
-    "train_loss": 1.4444872824584736,
-    "train_runtime": 72477.8005,
+    "eval_samples_per_second": 6.508,
+    "eval_steps_per_second": 0.102,
+    "train_loss": 27788.47707232307,
+    "train_runtime": 72625.1448,
     "train_samples": 207865,
-    "train_samples_per_second": 1.924,
+    "train_samples_per_second": 1.92,
     "train_steps_per_second": 0.004
 }

config.json

@@ -3,6 +3,7 @@
   "architectures": [
     "MistralForCausalLM"
   ],
+  "attention_dropout": 0.0,
   "bos_token_id": 1,
   "eos_token_id": 2,
   "hidden_act": "silu",
@@ -19,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.35.0",
+  "transformers_version": "4.36.0",
   "use_cache": true,
   "vocab_size": 32000
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_loss": 1.0922234058380127,
-    "eval_runtime": 2385.5237,
+    "eval_loss": 11.680082321166992,
+    "eval_runtime": 2370.9651,
     "eval_samples": 23110,
-    "eval_samples_per_second": 6.469,
-    "eval_steps_per_second": 0.101
+    "eval_samples_per_second": 6.508,
+    "eval_steps_per_second": 0.102
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 1.4444872824584736,
-    "train_runtime": 72477.8005,
+    "train_loss": 27788.47707232307,
+    "train_runtime": 72625.1448,
     "train_samples": 207865,
-    "train_samples_per_second": 1.924,
+    "train_samples_per_second": 1.92,
     "train_steps_per_second": 0.004
 }

trainer_state.json

@@ -11,348 +11,348 @@
     {
       "epoch": 0.0,
       "learning_rate": 1.9999332998034515e-05,
-      "loss": 1.9596,
+      "loss": 27792.8418,
       "step": 1
     },
     {
       "epoch": 0.02,
       "learning_rate": 1.99833293993636e-05,
-      "loss": 1.9397,
+      "loss": 27792.707,
       "step": 5
     },
     {
       "epoch": 0.04,
       "learning_rate": 1.99333731792395e-05,
-      "loss": 1.8704,
+      "loss": 27792.2687,
       "step": 10
     },
     {
       "epoch": 0.06,
       "learning_rate": 1.985029789966671e-05,
-      "loss": 1.7903,
+      "loss": 27791.6813,
       "step": 15
     },
     {
       "epoch": 0.07,
       "learning_rate": 1.9734380543606932e-05,
-      "loss": 1.6992,
+      "loss": 27790.8375,
       "step": 20
     },
     {
       "epoch": 0.09,
       "learning_rate": 1.9586007593450098e-05,
-      "loss": 1.6678,
+      "loss": 27790.5062,
       "step": 25
     },
     {
       "epoch": 0.11,
       "learning_rate": 1.9405673742435677e-05,
-      "loss": 1.6189,
+      "loss": 27790.025,
       "step": 30
     },
     {
       "epoch": 0.13,
       "learning_rate": 1.9193980245285967e-05,
-      "loss": 1.6122,
+      "loss": 27790.0,
       "step": 35
     },
     {
       "epoch": 0.15,
       "learning_rate": 1.8951632913550625e-05,
-      "loss": 1.5687,
+      "loss": 27789.6,
       "step": 40
     },
     {
       "epoch": 0.17,
       "learning_rate": 1.8679439762346186e-05,
-      "loss": 1.5472,
+      "loss": 27789.4313,
       "step": 45
     },
     {
       "epoch": 0.18,
       "learning_rate": 1.8378308316336585e-05,
-      "loss": 1.5302,
+      "loss": 27789.3187,
       "step": 50
     },
     {
       "epoch": 0.2,
       "learning_rate": 1.8049242583936923e-05,
-      "loss": 1.5045,
+      "loss": 27789.1219,
       "step": 55
     },
     {
       "epoch": 0.22,
       "learning_rate": 1.769333970982879e-05,
-      "loss": 1.49,
+      "loss": 27789.0312,
       "step": 60
     },
     {
       "epoch": 0.24,
       "learning_rate": 1.7311786316948112e-05,
-      "loss": 1.4782,
+      "loss": 27788.9625,
       "step": 65
     },
     {
       "epoch": 0.26,
       "learning_rate": 1.6905854550141717e-05,
-      "loss": 1.4596,
+      "loss": 27788.8125,
       "step": 70
     },
     {
       "epoch": 0.28,
       "learning_rate": 1.647689783468362e-05,
-      "loss": 1.4509,
+      "loss": 27788.75,
       "step": 75
     },
     {
       "epoch": 0.29,
       "learning_rate": 1.6026346363792565e-05,
-      "loss": 1.4295,
+      "loss": 27788.5375,
       "step": 80
     },
     {
       "epoch": 0.31,
       "learning_rate": 1.5555702330196024e-05,
-      "loss": 1.4301,
+      "loss": 27788.5438,
       "step": 85
     },
     {
       "epoch": 0.33,
       "learning_rate": 1.5066534917639195e-05,
-      "loss": 1.4297,
+      "loss": 27788.5469,
       "step": 90
     },
     {
       "epoch": 0.35,
       "learning_rate": 1.4560475069037895e-05,
-      "loss": 1.4161,
+      "loss": 27788.3937,
       "step": 95
     },
     {
       "epoch": 0.37,
       "learning_rate": 1.403921004871895e-05,
-      "loss": 1.4144,
+      "loss": 27788.3688,
       "step": 100
     },
     {
       "epoch": 0.39,
       "learning_rate": 1.350447781687826e-05,
-      "loss": 1.4051,
+      "loss": 27788.2625,
       "step": 105
     },
     {
       "epoch": 0.4,
       "learning_rate": 1.2958061235012707e-05,
-      "loss": 1.4061,
+      "loss": 27788.2594,
       "step": 110
     },
     {
       "epoch": 0.42,
       "learning_rate": 1.2401782121645767e-05,
-      "loss": 1.3968,
+      "loss": 27788.1562,
       "step": 115
     },
     {
       "epoch": 0.44,
       "learning_rate": 1.1837495178165706e-05,
-      "loss": 1.3946,
+      "loss": 27788.1188,
       "step": 120
     },
     {
       "epoch": 0.46,
       "learning_rate": 1.126708180502834e-05,
-      "loss": 1.382,
+      "loss": 27787.9625,
       "step": 125
     },
     {
       "epoch": 0.48,
       "learning_rate": 1.0692443828941918e-05,
-      "loss": 1.3844,
+      "loss": 27787.9813,
       "step": 130
     },
     {
       "epoch": 0.5,
       "learning_rate": 1.0115497161948409e-05,
-      "loss": 1.3857,
+      "loss": 27787.9781,
       "step": 135
     },
     {
       "epoch": 0.51,
       "learning_rate": 9.538165413542607e-06,
-      "loss": 1.3812,
+      "loss": 27787.9188,
       "step": 140
     },
     {
       "epoch": 0.53,
       "learning_rate": 8.962373477126983e-06,
-      "loss": 1.3657,
+      "loss": 27787.7438,
       "step": 145
     },
     {
       "epoch": 0.55,
       "learning_rate": 8.39004111218587e-06,
-      "loss": 1.3757,
+      "loss": 27787.8438,
       "step": 150
     },
     {
       "epoch": 0.57,
       "learning_rate": 7.823076543576718e-06,
-      "loss": 1.3713,
+      "loss": 27787.7812,
       "step": 155
     },
     {
       "epoch": 0.59,
       "learning_rate": 7.263370099279173e-06,
-      "loss": 1.3653,
+      "loss": 27787.7031,
       "step": 160
     },
     {
       "epoch": 0.61,
       "learning_rate": 6.712787907814542e-06,
-      "loss": 1.3749,
+      "loss": 27787.7812,
       "step": 165
     },
     {
       "epoch": 0.62,
       "learning_rate": 6.173165676349103e-06,
-      "loss": 1.3718,
+      "loss": 27787.7594,
       "step": 170
     },
     {
       "epoch": 0.64,
       "learning_rate": 5.646302570225919e-06,
-      "loss": 1.3728,
+      "loss": 27787.7594,
       "step": 175
     },
     {
       "epoch": 0.66,
       "learning_rate": 5.133955214331439e-06,
-      "loss": 1.3672,
+      "loss": 27787.7062,
       "step": 180
     },
     {
       "epoch": 0.68,
       "learning_rate": 4.637831836297103e-06,
-      "loss": 1.366,
+      "loss": 27787.6844,
       "step": 185
     },
     {
       "epoch": 0.7,
       "learning_rate": 4.1595865710632366e-06,
-      "loss": 1.3708,
+      "loss": 27787.725,
       "step": 190
     },
     {
       "epoch": 0.72,
       "learning_rate": 3.700813945794425e-06,
-      "loss": 1.37,
+      "loss": 27787.7125,
       "step": 195
     },
     {
       "epoch": 0.73,
       "learning_rate": 3.2630435635344283e-06,
-      "loss": 1.3679,
+      "loss": 27787.7,
       "step": 200
     },
     {
       "epoch": 0.75,
       "learning_rate": 2.847735003325868e-06,
-      "loss": 1.3671,
+      "loss": 27787.6813,
       "step": 205
     },
     {
       "epoch": 0.77,
       "learning_rate": 2.456272953798361e-06,
-      "loss": 1.3677,
+      "loss": 27787.6875,
       "step": 210
     },
     {
       "epoch": 0.79,
       "learning_rate": 2.0899625964503113e-06,
-      "loss": 1.3675,
+      "loss": 27787.675,
       "step": 215
     },
     {
       "epoch": 0.81,
       "learning_rate": 1.7500252540169782e-06,
-      "loss": 1.3647,
+      "loss": 27787.6437,
       "step": 220
     },
     {
       "epoch": 0.83,
       "learning_rate": 1.4375943184337871e-06,
-      "loss": 1.371,
+      "loss": 27787.7188,
       "step": 225
     },
     {
       "epoch": 0.84,
       "learning_rate": 1.1537114719714482e-06,
-      "loss": 1.3622,
+      "loss": 27787.6188,
       "step": 230
     },
     {
       "epoch": 0.86,
       "learning_rate": 8.993232141421415e-07,
-      "loss": 1.3596,
+      "loss": 27787.5938,
       "step": 235
     },
     {
       "epoch": 0.88,
       "learning_rate": 6.752777059564431e-07,
-      "loss": 1.369,
+      "loss": 27787.6937,
       "step": 240
     },
     {
       "epoch": 0.9,
       "learning_rate": 4.823219420526182e-07,
-      "loss": 1.3621,
+      "loss": 27787.6219,
       "step": 245
     },
     {
       "epoch": 0.92,
       "learning_rate": 3.2109926012677484e-07,
-      "loss": 1.3649,
+      "loss": 27787.6437,
       "step": 250
     },
     {
       "epoch": 0.94,
       "learning_rate": 1.921471959676957e-07,
-      "loss": 1.3646,
+      "loss": 27787.6469,
       "step": 255
     },
     {
       "epoch": 0.95,
       "learning_rate": 9.589569124794918e-08,
-      "loss": 1.3552,
+      "loss": 27787.55,
       "step": 260
     },
     {
       "epoch": 0.97,
       "learning_rate": 3.266566004670013e-08,
-      "loss": 1.3669,
+      "loss": 27787.6688,
       "step": 265
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.667918883627607e-09,
-      "loss": 1.3681,
+      "loss": 27787.675,
       "step": 270
     },
     {
       "epoch": 1.0,
-      "eval_loss": 1.0922234058380127,
-      "eval_runtime": 2386.6161,
-      "eval_samples_per_second": 6.466,
-      "eval_steps_per_second": 0.101,
+      "eval_loss": 11.680082321166992,
+      "eval_runtime": 2372.6436,
+      "eval_samples_per_second": 6.504,
+      "eval_steps_per_second": 0.102,
       "step": 272
     },
     {
       "epoch": 1.0,
       "step": 272,
       "total_flos": 7631468079611904.0,
-      "train_loss": 1.4444872824584736,
-      "train_runtime": 72477.8005,
-      "train_samples_per_second": 1.924,
+      "train_loss": 27788.47707232307,
+      "train_runtime": 72625.1448,
+      "train_samples_per_second": 1.92,
       "train_steps_per_second": 0.004
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c87469285d4d700c58756e9373aa90df9e87013e22d4b14969e26b47131d2b4
+oid sha256:994b25acdd78712b39ad725d07d1f583224f5b77b9fb320ef04258ede959fd42
 size 5752