cache_dir: ./cache
ddp_find_unused_parameters: false
ddp_timeout: 30000
device_map: auto
do_eval: true
do_train: true
eval_steps: 1000
evaluation_strategy: steps
fp16: true
gradient_accumulation_steps: 1
gradient_checkpointing: true
gradient_checkpointing_kwargs:
  use_reentrant: false
hub_model_id: hllj/sft-mistral-v1-clean-valid
hub_strategy: every_save
learning_rate: 3.0e-05
log_level: info
logging_first_step: true
logging_steps: 10
logging_strategy: steps
lora_alpha: 128
lora_dropout: 0.05
lora_r: 256
lora_target_modules:
- q_proj
- k_proj
- v_proj
- o_proj
lr_scheduler_type: cosine
max_seq_length: 1024
model_name_or_path: hllj/mistral-vi-math
model_type: auto
num_train_epochs: 2
output_dir: outputs-sft-mistral-v1-clean-valid
overwrite_output_dir: true
per_device_eval_batch_size: 4
per_device_train_batch_size: 4
preprocessing_num_workers: 4
push_to_hub: true
report_to: wandb
run_name: sft-mistral-v1-clean-valid
save_steps: 1000
save_strategy: steps
save_total_limit: 13
seed: 42
token: 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
torch_dtype: float16
train_file_dir: datasets/finetune
use_peft: true
validation_file_dir: datasets/validation
warmup_ratio: 0.05
weight_decay: 0.05