Update README.md

README.md

@@ -11,8 +11,8 @@ base_model: microsoft/Phi-3-mini-4k-instruct
 model_type: AutoModelForCausalLM
 tokenizer_type: AutoTokenizer
 trust_remote_code: True
-load_in_8bit: false
-load_in_4bit: true
+load_in_8bit: true
+load_in_4bit: false
 strict: false
 sequence_len: 4096
 bf16: auto
@@ -24,9 +24,10 @@ flash_attention: true
 datasets:
   - path: macadeliccc/opus_samantha
     type: sharegpt
+    conversation: chatml
 
 # Iterations
-num_epochs: 2
+num_epochs: 3
 
 # Evaluation
 val_set_size: 0.05
@@ -37,8 +38,8 @@ eval_sample_packing: false
 eval_batch_size: 1
 
 # LoRA
-output_dir: ./qlora-out
-adapter: qlora
+output_dir: ./lora-out
+adapter: lora
 lora_model_dir:
 lora_r: 32
 lora_alpha: 16
@@ -47,6 +48,10 @@ lora_target_linear: true
 lora_fan_in_fan_out:
 lora_target_modules:
 
+lora_modules_to_save:
+    - embed_tokens
+    - lm_head
+
 # Sampling
 sample_packing: false
 pad_to_sequence_len: false
@@ -77,6 +82,13 @@ deepspeed:
 weight_decay: 0
 fsdp:
 fsdp_config:
+special_tokens:
+  bos_token: "<s>"
+  eos_token: "</s>"
+  unk_token: "<unk>"
+tokens: # these are delimiters
+  - "<|im_start|>"
+  - "<|im_end|>"
 ```
 
 [<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)