Commit
•
73431f1
1
Parent(s):
dedb681
Update README.md
Browse files
README.md
CHANGED
|
@@ -46,4 +46,12 @@ Training settings:
|
|
| 46 |
- lr_scheduler_type: linear
|
| 47 |
- warmup_ratio: 0.06
|
| 48 |
- weight_decay: 0.1
|
| 49 |
-
- optimizer: adamw_torch_fused
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
- lr_scheduler_type: linear
|
| 47 |
- warmup_ratio: 0.06
|
| 48 |
- weight_decay: 0.1
|
| 49 |
+
- optimizer: adamw_torch_fused
|
| 50 |
+
|
| 51 |
+
LoRA config:
|
| 52 |
+
- target_modules: ['q_proj', 'k_proj', 'v_proj', 'o_proj']
|
| 53 |
+
- r: 64
|
| 54 |
+
- lora_alpha: 32
|
| 55 |
+
- lora_dropout: 0.05
|
| 56 |
+
- bias: "none"
|
| 57 |
+
- task_type: "CAUSAL_LM"
|