update model weights

cfg.yaml

@@ -1,6 +1,5 @@
 architecture:
     backbone_dtype: bfloat16
-    force_embedding_gradients: false
     gradient_checkpointing: true
     intermediate_dropout: 0.0
     pretrained: true
@@ -27,19 +26,19 @@ dataset:
     personalize: false
     prompt_column:
     - instruction
-    system_column: None
+    system_column: system
     text_answer_separator: <|answer|>
     text_prompt_start: <|prompt|>
     text_system_start: <|system|>
-    train_dataframe: /home/qishen/src/h2o-llmstudio/data/user/japanese_hh-rlhf-49k/train-00000-of-00001-157934b4864eb8e0.parquet
+    train_dataframe: /home/user/src/h2o-llmstudio/data/user/japanese_hh-rlhf-49k/japanese_hh-rlhf-49k.csv
     validation_dataframe: None
     validation_size: 0.01
     validation_strategy: automatic
 environment:
     compile_model: false
-    deepspeed_allgather_bucket_size: 500000000
+    deepspeed_allgather_bucket_size: 100000000
     deepspeed_method: ZeRO2
-    deepspeed_reduce_bucket_size: 500000000
+    deepspeed_reduce_bucket_size: 100000000
     deepspeed_stage3_param_persistence_threshold: 1000000
     deepspeed_stage3_prefetch_bucket_size: 1000000
     find_unused_parameters: false
@@ -47,22 +46,23 @@ environment:
     - '0'
     - '1'
     huggingface_branch: main
-    mixed_precision: true
+    mixed_precision: false
+    mixed_precision_dtype: bfloat16
     number_of_workers: 8
     seed: -1
     trust_remote_code: true
     use_deepspeed: true
-experiment_name: Llama-3-8B-Instruct
+experiment_name: llama-3-8b-ja
 llm_backbone: meta-llama/Meta-Llama-3-8B-Instruct
 logging:
     logger: None
     neptune_project: ''
-output_directory: /home/qishen/src/h2o-llmstudio/output/user/Llama-3-8B-Instruct/
+output_directory: /home/user/src/h2o-llmstudio/output/user/llama-3-8b-ja/
 prediction:
     batch_size_inference: 0
     do_sample: false
-    max_length_inference: 256
-    max_time: 120.0
+    max_length_inference: 512
+    max_time: 0.0
     metric: Perplexity
     metric_gpt_model: gpt-3.5-turbo-0301
     metric_gpt_template: general
@@ -77,11 +77,9 @@ prediction:
 problem_type: text_causal_language_modeling
 tokenizer:
     add_prompt_answer_tokens: false
-    max_length: 8160
-    max_length_answer: 4064
-    max_length_prompt: 4096
+    max_length: 1024
     padding_quantile: 1.0
-    use_fast: true
+    tokenizer_kwargs: '{"use_fast": true, "add_prefix_space": false}'
 training:
     batch_size: 2
     differential_learning_rate: 1.0e-05
@@ -90,19 +88,22 @@ training:
     epochs: 1
     evaluate_before_training: false
     evaluation_epochs: 1.0
-    grad_accumulation: 1
+    freeze_layers: []
+    grad_accumulation: 4
     gradient_clip: 0.0
-    learning_rate: 0.0001
+    learning_rate: 1.0e-05
     lora: true
     lora_alpha: 16
     lora_dropout: 0.05
     lora_r: 4
-    lora_target_modules: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj
+    lora_target_modules: ''
+    lora_unfreeze_layers: []
     loss_function: TokenAveragedCrossEntropy
     optimizer: AdamW
-    save_best_checkpoint: false
+    save_checkpoint: last
     schedule: Cosine
     train_validation_data: false
-    use_flash_attention_2: false
-    warmup_epochs: 0.0
+    use_dora: false
+    use_flash_attention_2: true
+    warmup_epochs: 0.05
     weight_decay: 0.0

generation_config.json

@@ -1,10 +1,10 @@
 {
   "_from_model_config": true,
   "bos_token_id": 128000,
-  "eos_token_id": 128001,
-  "max_new_tokens": 256,
-  "max_time": 120.0,
-  "min_new_tokens": 2,
-  "pad_token_id": 128001,
-  "transformers_version": "4.38.2"
+  "eos_token_id": 128009,
+  "pad_token_id": 128009,
+  "temperature": null,
+  "top_k": null,
+  "top_p": null,
+  "transformers_version": "4.40.2"
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:61f4c8da26c66d743d1bd7815b2970c7ced7440f1c78cd6b8f386c8c1ebf9a5b
+oid sha256:97f086559f12923ca1b59a1c1c0367d4b0913ab6bfc66efd687af54e5dff8a41
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37c782e9bd0d0b1456039b3167ba69a9872cffc263af643e522b615136cf34ac
+oid sha256:ac035c87cda3f09cc332ab50ec588537b44b447d165a81fb92d510155198b777
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7633ed632f3d6ffef34ce6d1a0f0fc978a1a96f4a1030d91939ebc562629f056
+oid sha256:99313f6223273a020ecc15a3a249307eb34e49ef4ca24ce60982cda6430c7272
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f006959be3288f0af300a0f6bda45f98e14237fbed62da6d8994656cfff21f2
+oid sha256:1c7049e9604e4838318948587dd94b0f31cd101a4a90529ec266540ac1d2035f
 size 1168138808