cutoff_len: 2048 | |
dataset: farcaster_pretrain | |
dataset_dir: data | |
ddp_timeout: 180000000 | |
do_train: true | |
finetuning_type: lora | |
flash_attn: auto | |
fp16: true | |
gradient_accumulation_steps: 8 | |
include_num_input_tokens_seen: true | |
learning_rate: 0.0002 | |
logging_steps: 1 | |
lora_alpha: 16 | |
lora_dropout: 0 | |
lora_rank: 256 | |
lora_target: all | |
lr_scheduler_type: cosine | |
max_grad_norm: 1.0 | |
max_samples: 100000 | |
model_name_or_path: Qwen/Qwen2-7B-Instruct | |
num_train_epochs: 8.0 | |
optim: adamw_8bit | |
output_dir: saves/Qwen2-7B-Chat/lora/final-pretrain-v2 | |
packing: true | |
per_device_train_batch_size: 1 | |
plot_loss: true | |
preprocessing_num_workers: 16 | |
report_to: none | |
save_steps: 100 | |
stage: pt | |
template: qwen | |
warmup_steps: 0 | |