End of training

README.md

@@ -2,8 +2,6 @@
 license: apache-2.0
 library_name: peft
 tags:
-- trl
-- sft
 - generated_from_trainer
 base_model: vilsonrodrigues/falcon-7b-instruct-sharded
 model-index:
@@ -36,15 +34,15 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
-- train_batch_size: 16
+- train_batch_size: 1
 - eval_batch_size: 8
 - seed: 42
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 64
+- total_train_batch_size: 4
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
-- lr_scheduler_warmup_ratio: 0.03
-- training_steps: 180
+- lr_scheduler_warmup_ratio: 0.05
+- num_epochs: 10
 - mixed_precision_training: Native AMP
 
 ### Training results

adapter_config.json

@@ -15,14 +15,11 @@
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 32,
+  "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "dense_h_to_4h",
-    "query_key_value",
-    "dense",
-    "dense_4h_to_h"
+    "query_key_value"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86055f23f572df48893b4b5e540fd12062763ec5aa10b09e786c6fb99e1d8217
-size 261131840
+oid sha256:3c941424e554bc50c9bceb7f9ee5d873bd619a4565fb7a6d9f127ce97f4cbc8f
+size 18883912

runs/Feb21_08-55-21_512faa0505b2/events.out.tfevents.1708505722.512faa0505b2.5658.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a880b88705358113bac8359738dcfd3af20e4e0e9c6c04e5bfd1177cc110270
+size 12468

runs/Feb21_08-57-40_512faa0505b2/events.out.tfevents.1708505861.512faa0505b2.5658.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7029a668708f6a855cde142f52c03e65cb7201ebab94f047b9aa68ce100821f2
+size 37459

runs/Feb21_09-05-59_512faa0505b2/events.out.tfevents.1708506359.512faa0505b2.5658.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db3fb15d3799f88c872469f10c971552d22e8e0ec6a4481a5d3d1df415ef6234
+size 72982

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27b4fe67526c2ab95e4dc03472a71781455227508a77bc5eef51bc3f27ee6b59
+oid sha256:de9d00e9d163f1d242baf776dbaca2ea6a80ee779389307f42d889e63c5d4cbe
 size 4728