llama2-13B-supervised-ft-5epochs-411

README.md

@@ -4,7 +4,7 @@ tags:
 - trl
 - sft
 - generated_from_trainer
-base_model: Trelis/Llama-2-7b-chat-hf-sharded-bf16
+base_model: LLama_weights/tmp
 model-index:
 - name: outputs
   results: []
@@ -15,9 +15,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # outputs
 
-This model is a fine-tuned version of [Trelis/Llama-2-7b-chat-hf-sharded-bf16](https://huggingface.co/Trelis/Llama-2-7b-chat-hf-sharded-bf16) on the None dataset.
+This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.1177
+- Loss: 2.1290
 
 ## Model description
 
@@ -40,44 +40,31 @@ The following hyperparameters were used during training:
 - train_batch_size: 1
 - eval_batch_size: 1
 - seed: 42
-- gradient_accumulation_steps: 16
-- total_train_batch_size: 16
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- num_epochs: 5
+- num_epochs: 1
 - mixed_precision_training: Native AMP
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 2.3322        | 0.24  | 4    | 2.3370          |
-| 2.2294        | 0.47  | 8    | 2.3072          |
-| 2.343         | 0.71  | 12   | 2.2808          |
-| 2.3434        | 0.94  | 16   | 2.2575          |
-| 2.3287        | 1.18  | 20   | 2.2372          |
-| 2.189         | 1.41  | 24   | 2.2194          |
-| 2.2015        | 1.65  | 28   | 2.2040          |
-| 2.1622        | 1.88  | 32   | 2.1907          |
-| 2.1723        | 2.12  | 36   | 2.1790          |
-| 2.1742        | 2.35  | 40   | 2.1687          |
-| 2.0336        | 2.59  | 44   | 2.1597          |
-| 2.1124        | 2.82  | 48   | 2.1518          |
-| 2.1201        | 3.06  | 52   | 2.1449          |
-| 2.1692        | 3.29  | 56   | 2.1387          |
-| 2.2336        | 3.53  | 60   | 2.1334          |
-| 2.0764        | 3.76  | 64   | 2.1288          |
-| 2.1874        | 4.0   | 68   | 2.1250          |
-| 2.1178        | 4.24  | 72   | 2.1220          |
-| 2.1511        | 4.47  | 76   | 2.1199          |
-| 2.1382        | 4.71  | 80   | 2.1184          |
-| 2.1022        | 4.94  | 84   | 2.1177          |
+| 2.3285        | 0.12  | 4    | 2.1680          |
+| 2.2299        | 0.24  | 8    | 2.1595          |
+| 2.0283        | 0.35  | 12   | 2.1514          |
+| 2.3197        | 0.47  | 16   | 2.1442          |
+| 2.1502        | 0.59  | 20   | 2.1382          |
+| 2.1846        | 0.71  | 24   | 2.1336          |
+| 2.2649        | 0.82  | 28   | 2.1306          |
+| 2.252         | 0.94  | 32   | 2.1290          |
 
 
 ### Framework versions
 
 - PEFT 0.8.2
-- Transformers 4.38.0.dev0
-- Pytorch 2.1.0+cu121
-- Datasets 2.16.1
+- Transformers 4.37.2
+- Pytorch 2.2.0+cu118
+- Datasets 2.17.0
 - Tokenizers 0.15.1

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "Trelis/Llama-2-7b-chat-hf-sharded-bf16",
+  "base_model_name_or_path": "LLama_weights/tmp",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -9,23 +9,23 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 64,
+  "lora_alpha": 32,
   "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 32,
+  "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
     "o_proj",
     "up_proj",
-    "q_proj",
-    "down_proj",
+    "v_proj",
+    "gate_proj",
     "k_proj",
-    "v_proj"
+    "q_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45b285036211d3480b33717cc2cd70493d538455f8372fd4d1932dbf82b9a67e
-size 319876032
+oid sha256:5ca6b86293e72e448256ba721c6cb6d98e35653723bb8c580906d1823fe13d7e
+size 1001465824

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 3200,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {

tokenizer_config.json

@@ -38,12 +38,12 @@
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "legacy": false,
+  "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "[PAD]",
   "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false,
-  "use_fast": false
+  "use_default_system_prompt": false
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9fee88b62fd20e4d5eb24648a8c3a2fcab0d481839c0fc33b7b99a961ed4cd6
-size 4728
+oid sha256:665d0f5d76ff011650e9b363c0eeb92bde61fffd61ab1cd09a959010d6ff6457
+size 4664