Upload cfg.yaml

cfg.yaml

@@ -1,14 +1,14 @@
 architecture:
-    backbone_dtype: int8
+    backbone_dtype: int4
     force_embedding_gradients: true
     gradient_checkpointing: true
     intermediate_dropout: 0.0
     pretrained: true
-    pretrained_weights: /media/akshay/datasets/largeModels/llms/h2o/h2o-llmstudio/output/user/economic-ferret.1/checkpoint.pth
+    pretrained_weights: /media/akshay/datasets/largeModels/llms/h2o/h2o-llmstudio/output/user/raspberry-crocodile.1.2.1.1.1/checkpoint.pth
 augmentation:
-    random_parent_probability: 0.5
-    skip_parent_probability: 0.0
-    token_mask_probability: 0.0
+    random_parent_probability: 0.0
+    skip_parent_probability: 0.05
+    token_mask_probability: 0.3
 dataset:
     add_eos_token_to_answer: true
     add_eos_token_to_prompt: true
@@ -22,7 +22,7 @@ dataset:
     - Validation
     limit_chained_samples: false
     mask_prompt_labels: true
-    parent_id_column: None
+    parent_id_column: parent_id
     personalize: false
     prompt_column:
     - instruction
@@ -30,7 +30,7 @@ dataset:
     text_answer_separator: <|answer|>
     text_prompt_start: <|prompt|>
     text_system_start: <|system|>
-    train_dataframe: /media/akshay/datasets/largeModels/llms/h2o/h2o-llmstudio/data/user/PR-singleQA-July13/singleQA.csv
+    train_dataframe: /media/akshay/datasets/largeModels/llms/h2o/h2o-llmstudio/data/user/PR-Conversations-July21/conversations.csv
     validation_dataframe: None
     validation_size: 0.01
     validation_strategy: automatic
@@ -45,13 +45,13 @@ environment:
     seed: -1
     trust_remote_code: true
     use_fsdp: false
-experiment_name: economic-ferret.1.1
-llm_backbone: tiiuae/falcon-7b
+experiment_name: raspberry-crocodile-conv
+llm_backbone: tiiuae/falcon-7b-instruct
 logging:
     logger: None
     neptune_project: ''
     number_of_texts: 10
-output_directory: /media/akshay/datasets/largeModels/llms/h2o/h2o-llmstudio/output/user/economic-ferret.1.1/
+output_directory: /media/akshay/datasets/largeModels/llms/h2o/h2o-llmstudio/output/user/raspberry-crocodile-conv/
 prediction:
     batch_size_inference: 0
     do_sample: false
@@ -69,22 +69,22 @@ prediction:
 problem_type: text_causal_language_modeling
 tokenizer:
     add_prefix_space: false
-    add_prompt_answer_tokens: false
-    max_length: 1760
-    max_length_answer: 512
-    max_length_prompt: 1024
+    add_prompt_answer_tokens: true
+    max_length: 4096
+    max_length_answer: 2048
+    max_length_prompt: 2048
     padding_quantile: 1.0
     use_fast: true
 training:
     adaptive_kl_control: true
     advantages_gamma: 0.99
     advantages_lambda: 0.95
-    batch_size: 2
+    batch_size: 3
     differential_learning_rate: 1.0e-05
     differential_learning_rate_layers: []
     drop_last_batch: true
-    epochs: 3
-    evaluate_before_training: true
+    epochs: 4
+    evaluate_before_training: false
     evaluation_epochs: 1.0
     grad_accumulation: 4
     gradient_clip: 0.9
@@ -96,7 +96,7 @@ training:
     lora_alpha: 16
     lora_dropout: 0.05
     lora_r: 8
-    lora_target_modules: query_key_value, dense, dense_h_to_4h, dense_4h_to_h
+    lora_target_modules: ''
     loss_function: TokenAveragedCrossEntropy
     offload_reward_model: false
     optimizer: AdamW