Training in progress, step 400

axolotl_config.yaml

@@ -13,7 +13,9 @@ datasets:
     no_input_format: '{instruction}'
     system_format: '{system}'
     system_prompt: ''
-eval_steps: 20
+device_map: auto
+eval_sample_packing: false
+eval_steps: 200
 flash_attention: true
 gpu_memory_limit: 80GiB
 group_by_length: true
@@ -23,17 +25,19 @@ learning_rate: 0.0002
 logging_steps: 10
 lr_scheduler: cosine
 max_steps: 2500
-micro_batch_size: 2
+micro_batch_size: 4
 model_type: AutoModelForCausalLM
 num_epochs: 100
 optimizer: adamw_bnb_8bit
 output_dir: /root/.sn56/axolotl/outputs/tiny-random-LlamaForCausalLM
 pad_to_sequence_len: true
 resize_token_embeddings_to_32x: false
-save_steps: 40
+sample_packing: false
+save_steps: 400
 save_total_limit: 1
 sequence_len: 2048
 tokenizer_type: LlamaTokenizerFast
+torch_dtype: bf16
 trust_remote_code: true
 val_set_size: 0.1
 wandb_entity: ''

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d9f1e60d34df70b7758959048fe7f2701ecbcbb16b25defdf06c629d7d60c03
+oid sha256:deb58d744f23dab866f30a5eccd5ee89f1d373670d7096630c72465e9c1419c7
 size 2066752

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:374e5cd0f5c20c6c1df6bc90575a93f0b433af575c6b98fb295f25d134720cf9
-size 6776
+oid sha256:a7ab57c26802475df8b559ffa07b1995cecba20856adfc383ddb4700563cc1b9
+size 6904