Model save

README.md

@@ -5,9 +5,9 @@ tags:
 - trl
 - sft
 - generated_from_trainer
+base_model: meta-llama/Llama-2-7b-chat-hf
 datasets:
 - generator
-base_model: meta-llama/Llama-2-7b-chat-hf
 model-index:
 - name: llama2-7b-sft-lora
   results: []
@@ -42,18 +42,24 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 4
 - distributed_type: multi-GPU
-- num_devices: 4
+- num_devices: 2
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 64
-- total_eval_batch_size: 32
+- total_train_batch_size: 32
+- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - num_epochs: 1
 
 ### Framework versions
 
-- PEFT 0.8.2
-- Transformers 4.37.2
-- Pytorch 2.2.0+cu121
-- Datasets 2.16.1
-- Tokenizers 0.15.1
+- Transformers 4.41.0
+- Pytorch 2.1.0+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1
+## Training procedure
+
+
+### Framework versions
+
+
+- PEFT 0.6.1

adapter_config.json

@@ -8,24 +8,20 @@
   "init_lora_weights": true,
   "layers_pattern": null,
   "layers_to_transform": null,
-  "loftq_config": {},
   "lora_alpha": 16,
   "lora_dropout": 0.1,
-  "megatron_config": null,
-  "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
   "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "k_proj",
+    "down_proj",
     "v_proj",
     "q_proj",
-    "k_proj",
-    "o_proj",
     "up_proj",
-    "down_proj"
+    "o_proj"
   ],
-  "task_type": "CAUSAL_LM",
-  "use_rslora": false
+  "task_type": "CAUSAL_LM"
 }

runs/May20_14-41-33_04bac4264f87/events.out.tfevents.1716216487.04bac4264f87.3853.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36b1bd1bf287b0eaf3ea6ed62a68c8ce7b745fe4d536f214ea02b53c594e7735
+size 4955

step_0/README.md

@@ -81,7 +81,7 @@ Use the code below to get started with the model.
 
 [More Information Needed]
 
-### Training Procedure 
+### Training Procedure
 
 <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
 
@@ -197,8 +197,10 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 ## Model Card Contact
 
 [More Information Needed]
+## Training procedure
 
 
 ### Framework versions
 
-- PEFT 0.8.2
+
+- PEFT 0.6.1

step_0/adapter_config.json

@@ -8,24 +8,20 @@
   "init_lora_weights": true,
   "layers_pattern": null,
   "layers_to_transform": null,
-  "loftq_config": {},
   "lora_alpha": 16,
   "lora_dropout": 0.1,
-  "megatron_config": null,
-  "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
   "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "k_proj",
+    "down_proj",
     "v_proj",
     "q_proj",
-    "k_proj",
-    "o_proj",
     "up_proj",
-    "down_proj"
+    "o_proj"
   ],
-  "task_type": "CAUSAL_LM",
-  "use_rslora": false
+  "task_type": "CAUSAL_LM"
 }

step_0/tokenizer.json

@@ -134,6 +134,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

step_0/tokenizer_config.json

@@ -31,7 +31,6 @@
   "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "legacy": false,
   "model_max_length": 2048,
   "pad_token": "</s>",
   "padding_side": "right",

step_0/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a17890ae8adc5c75c8c626de8ca0b3b549dff571a11f82a797bc2d7a5b3bdbe
-size 5688
+oid sha256:5e053cf93f1736083927827130a127d281aa30f3d7b5c8bd7f0ef7a5ad298352
+size 6136

tokenizer.json

@@ -134,6 +134,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

tokenizer_config.json

@@ -31,7 +31,6 @@
   "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "legacy": false,
   "model_max_length": 2048,
   "pad_token": "</s>",
   "padding_side": "right",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a17890ae8adc5c75c8c626de8ca0b3b549dff571a11f82a797bc2d7a5b3bdbe
-size 5688
+oid sha256:5e053cf93f1736083927827130a127d281aa30f3d7b5c8bd7f0ef7a5ad298352
+size 6136