use scale factor 1 and rank 256

README.md

@@ -27,7 +27,7 @@ strict: false
 datasets:
   - path: Vezora/Tested-22k-Python-Alpaca
     type: alpaca
-dataset_prepared_path: prepared-galore
+dataset_prepared_path: prepared-qlora
 val_set_size: 0.05
 output_dir: ./galore-out
 
@@ -60,6 +60,9 @@ optim_target_modules:
   - self_attn  # for llama
   - mlp
 
+optim_args:
+  rank: 256
+  scale: 1
 
 train_on_inputs: false
 group_by_length: false
@@ -93,7 +96,7 @@ special_tokens:
 
 This model is a fine-tuned version of [HuggingFaceTB/cosmo-1b](https://huggingface.co/HuggingFaceTB/cosmo-1b) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.3833
+- Loss: 0.2426
 
 ## Model description
 
@@ -128,9 +131,9 @@ The following hyperparameters were used during training:
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
 | 0.6299        | 0.0   | 1    | 0.6469          |
-| 0.4194        | 0.25  | 217  | 0.4653          |
-| 0.3577        | 0.5   | 434  | 0.4068          |
-| 0.4144        | 0.75  | 651  | 0.3833          |
+| 0.4353        | 0.25  | 215  | 0.4139          |
+| 0.3721        | 0.5   | 430  | 0.2957          |
+| 0.3514        | 0.75  | 645  | 0.2426          |
 
 
 ### Framework versions

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:97d8deb89efe133140a772e95835a6051ca0ebc007f5b468fff984b0e9caa2ad
+oid sha256:1d6d4446971e36016f8a0edcef89d93b65b7256eeb5f29482fa1745a7ebef215
 size 3483644214