Update v2.yml

v2.yml

@@ -10,6 +10,8 @@ strict: false
 
 ## data 
 datasets:
+  - path: PocketDoc/Dans-Codemaxx-LeetCode
+    type: dan-chat-advanced
   - path: Nitral-AI/ARES-ShareGPT
     type: dan-chat-advanced
   - path: PocketDoc/Dans-Logicmaxx-FI-VeriMed
@@ -38,16 +40,10 @@ datasets:
     type: dan-chat-advanced
   - path: PocketDoc/Dans-Prosemaxx-Instructwriter-Long
     type: dan-chat-advanced
-  - path: PocketDoc/Dans-Personamaxx-VN
-    type: dan-chat-advanced
   - path: PocketDoc/Dans-Prosemaxx-Cowriter-3-XS
     type: dan-chat-advanced
   - path: PocketDoc/Dans-Prosemaxx-InstructWriter-ZeroShot
     type: dan-chat-advanced
-  - path: Delta-Vector/Orion-BlueSky-10K-Complexity
-    type: dan-chat-advanced
-  - path: Delta-Vector/Orion-Shoujo-AI-Filtered-ShareGPT
-    type: dan-chat-advanced
   - path: PocketDoc/Dans-Benchmaxx-COT
     type: dan-chat-advanced
   - path: PocketDoc/Dans-Benchmaxx
@@ -76,7 +72,7 @@ eval_sample_packing: false
 pad_to_sequence_len: true
 
 ## max grad norm
-max_grad_norm: 1.0
+max_grad_norm: 0.001
 
 
 ## WandB
@@ -92,14 +88,15 @@ eval_table_size:
 eval_max_new_tokens: 128
 
 ## hparams
-gradient_accumulation_steps: 2
-micro_batch_size: 2
-num_epochs: 2
+gradient_accumulation_steps: 6
+micro_batch_size: 6
+num_epochs: 4
 optimizer: paged_ademamix_8bit
-lr_scheduler: cosine
-learning_rate: 1e-5
-warmup_ratio: 0.2
-weight_decay: 0.0025
+optim_args: "beta1=0.9,beta2=0.999,beta3=0.999,alpha=5"
+lr_scheduler: rex
+learning_rate: 1e-6
+warmup_ratio: 0.1
+weight_decay: 0.0
 
 train_on_inputs: false
 group_by_length: false