Model save

README.md

@@ -0,0 +1,69 @@
+---
+library_name: peft
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.3
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: mistral7b-lora-coding-11-v1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral7b-lora-coding-11-v1
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.3](https://huggingface.co/mistralai/Mistral-7B-v0.3) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.3992
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 14
+- eval_batch_size: 14
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 224
+- total_eval_batch_size: 112
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.8942        | 1.0   | 185  | 1.3992          |
+
+
+### Framework versions
+
+- PEFT 0.13.2
+- Transformers 4.46.2
+- Pytorch 2.3.1+cu121
+- Datasets 3.1.0
+- Tokenizers 0.20.3

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 9.060485625492275e+17,
+    "train_loss": 0.9615903693276483,
+    "train_runtime": 3138.0854,
+    "train_samples": 116368,
+    "train_samples_per_second": 13.203,
+    "train_steps_per_second": 0.059
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 9.060485625492275e+17,
+    "train_loss": 0.9615903693276483,
+    "train_runtime": 3138.0854,
+    "train_samples": 116368,
+    "train_samples_per_second": 13.203,
+    "train_steps_per_second": 0.059
+}

trainer_state.json

@@ -0,0 +1,316 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 185,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005405405405405406,
+      "grad_norm": 2.6925952434539795,
+      "learning_rate": 1.0526315789473684e-05,
+      "loss": 1.4517,
+      "step": 1
+    },
+    {
+      "epoch": 0.02702702702702703,
+      "grad_norm": 2.4539754390716553,
+      "learning_rate": 5.2631578947368424e-05,
+      "loss": 1.4341,
+      "step": 5
+    },
+    {
+      "epoch": 0.05405405405405406,
+      "grad_norm": 2.1194558143615723,
+      "learning_rate": 0.00010526315789473685,
+      "loss": 1.3761,
+      "step": 10
+    },
+    {
+      "epoch": 0.08108108108108109,
+      "grad_norm": 2.0553150177001953,
+      "learning_rate": 0.00015789473684210527,
+      "loss": 1.2687,
+      "step": 15
+    },
+    {
+      "epoch": 0.10810810810810811,
+      "grad_norm": 1.8468493223190308,
+      "learning_rate": 0.00019998209226697376,
+      "loss": 1.1537,
+      "step": 20
+    },
+    {
+      "epoch": 0.13513513513513514,
+      "grad_norm": 1.0270365476608276,
+      "learning_rate": 0.0001993559947963185,
+      "loss": 1.0561,
+      "step": 25
+    },
+    {
+      "epoch": 0.16216216216216217,
+      "grad_norm": 0.9126940369606018,
+      "learning_rate": 0.00019784091409455728,
+      "loss": 0.9985,
+      "step": 30
+    },
+    {
+      "epoch": 0.1891891891891892,
+      "grad_norm": 0.6563550233840942,
+      "learning_rate": 0.0001954504062771555,
+      "loss": 0.9701,
+      "step": 35
+    },
+    {
+      "epoch": 0.21621621621621623,
+      "grad_norm": 0.6633013486862183,
+      "learning_rate": 0.00019220586030376134,
+      "loss": 0.9619,
+      "step": 40
+    },
+    {
+      "epoch": 0.24324324324324326,
+      "grad_norm": 0.7960276007652283,
+      "learning_rate": 0.00018813630660146488,
+      "loss": 0.9392,
+      "step": 45
+    },
+    {
+      "epoch": 0.2702702702702703,
+      "grad_norm": 0.6284172534942627,
+      "learning_rate": 0.00018327815731637612,
+      "loss": 0.9387,
+      "step": 50
+    },
+    {
+      "epoch": 0.2972972972972973,
+      "grad_norm": 0.6241137981414795,
+      "learning_rate": 0.00017767488051760857,
+      "loss": 0.9316,
+      "step": 55
+    },
+    {
+      "epoch": 0.32432432432432434,
+      "grad_norm": 0.7423997521400452,
+      "learning_rate": 0.0001713766112687139,
+      "loss": 0.9287,
+      "step": 60
+    },
+    {
+      "epoch": 0.35135135135135137,
+      "grad_norm": 0.6817660331726074,
+      "learning_rate": 0.0001644397030464877,
+      "loss": 0.9293,
+      "step": 65
+    },
+    {
+      "epoch": 0.3783783783783784,
+      "grad_norm": 0.6054185628890991,
+      "learning_rate": 0.00015692622352080662,
+      "loss": 0.9134,
+      "step": 70
+    },
+    {
+      "epoch": 0.40540540540540543,
+      "grad_norm": 0.6050704121589661,
+      "learning_rate": 0.00014890339920698334,
+      "loss": 0.93,
+      "step": 75
+    },
+    {
+      "epoch": 0.43243243243243246,
+      "grad_norm": 0.589340090751648,
+      "learning_rate": 0.0001404430139595877,
+      "loss": 0.9163,
+      "step": 80
+    },
+    {
+      "epoch": 0.4594594594594595,
+      "grad_norm": 0.6608156561851501,
+      "learning_rate": 0.0001316207666896824,
+      "loss": 0.9211,
+      "step": 85
+    },
+    {
+      "epoch": 0.4864864864864865,
+      "grad_norm": 0.5910987854003906,
+      "learning_rate": 0.00012251559405226941,
+      "loss": 0.9081,
+      "step": 90
+    },
+    {
+      "epoch": 0.5135135135135135,
+      "grad_norm": 0.6293689608573914,
+      "learning_rate": 0.00011320896416417026,
+      "loss": 0.9123,
+      "step": 95
+    },
+    {
+      "epoch": 0.5405405405405406,
+      "grad_norm": 0.5938843488693237,
+      "learning_rate": 0.00010378414767176705,
+      "loss": 0.902,
+      "step": 100
+    },
+    {
+      "epoch": 0.5675675675675675,
+      "grad_norm": 0.6008319854736328,
+      "learning_rate": 9.432547269069261e-05,
+      "loss": 0.9061,
+      "step": 105
+    },
+    {
+      "epoch": 0.5945945945945946,
+      "grad_norm": 0.6468297839164734,
+      "learning_rate": 8.491757028386263e-05,
+      "loss": 0.9038,
+      "step": 110
+    },
+    {
+      "epoch": 0.6216216216216216,
+      "grad_norm": 0.6693244576454163,
+      "learning_rate": 7.564461722890081e-05,
+      "loss": 0.911,
+      "step": 115
+    },
+    {
+      "epoch": 0.6486486486486487,
+      "grad_norm": 0.590085506439209,
+      "learning_rate": 6.658958285026102e-05,
+      "loss": 0.9041,
+      "step": 120
+    },
+    {
+      "epoch": 0.6756756756756757,
+      "grad_norm": 0.5497063398361206,
+      "learning_rate": 5.7833486654981606e-05,
+      "loss": 0.9066,
+      "step": 125
+    },
+    {
+      "epoch": 0.7027027027027027,
+      "grad_norm": 0.5866259932518005,
+      "learning_rate": 4.945467341434195e-05,
+      "loss": 0.904,
+      "step": 130
+    },
+    {
+      "epoch": 0.7297297297297297,
+      "grad_norm": 0.6213558316230774,
+      "learning_rate": 4.152811217759529e-05,
+      "loss": 0.9036,
+      "step": 135
+    },
+    {
+      "epoch": 0.7567567567567568,
+      "grad_norm": 0.5839004516601562,
+      "learning_rate": 3.4124725489820645e-05,
+      "loss": 0.8903,
+      "step": 140
+    },
+    {
+      "epoch": 0.7837837837837838,
+      "grad_norm": 0.6068409085273743,
+      "learning_rate": 2.7310754815685624e-05,
+      "loss": 0.8978,
+      "step": 145
+    },
+    {
+      "epoch": 0.8108108108108109,
+      "grad_norm": 0.6482556462287903,
+      "learning_rate": 2.1147167846963422e-05,
+      "loss": 0.8958,
+      "step": 150
+    },
+    {
+      "epoch": 0.8378378378378378,
+      "grad_norm": 0.5723783373832703,
+      "learning_rate": 1.5689112996891576e-05,
+      "loss": 0.9028,
+      "step": 155
+    },
+    {
+      "epoch": 0.8648648648648649,
+      "grad_norm": 0.655846357345581,
+      "learning_rate": 1.0985425962260343e-05,
+      "loss": 0.8999,
+      "step": 160
+    },
+    {
+      "epoch": 0.8918918918918919,
+      "grad_norm": 0.6444997787475586,
+      "learning_rate": 7.078192768243486e-06,
+      "loss": 0.8923,
+      "step": 165
+    },
+    {
+      "epoch": 0.918918918918919,
+      "grad_norm": 0.5838979482650757,
+      "learning_rate": 4.002373205607723e-06,
+      "loss": 0.8973,
+      "step": 170
+    },
+    {
+      "epoch": 0.9459459459459459,
+      "grad_norm": 0.6335191130638123,
+      "learning_rate": 1.7854880295797405e-06,
+      "loss": 0.8826,
+      "step": 175
+    },
+    {
+      "epoch": 0.972972972972973,
+      "grad_norm": 0.5943057537078857,
+      "learning_rate": 4.4737271914411236e-07,
+      "loss": 0.8931,
+      "step": 180
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.600213885307312,
+      "learning_rate": 0.0,
+      "loss": 0.8942,
+      "step": 185
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 1.3991833925247192,
+      "eval_runtime": 1.4887,
+      "eval_samples_per_second": 7.389,
+      "eval_steps_per_second": 0.672,
+      "step": 185
+    },
+    {
+      "epoch": 1.0,
+      "step": 185,
+      "total_flos": 9.060485625492275e+17,
+      "train_loss": 0.9615903693276483,
+      "train_runtime": 3138.0854,
+      "train_samples_per_second": 13.203,
+      "train_steps_per_second": 0.059
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 185,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.060485625492275e+17,
+  "train_batch_size": 14,
+  "trial_name": null,
+  "trial_params": null
+}