|
base model = beomi-Llama-3-Open-Ko-8B-Instruct-preview |
|
base model = hansoldeco-beomi-Llama-3-Open-Ko-8B-Instruct-preview (Trained via Axolotl) |
|
|
|
dora_train config |
|
(from fsdp_qlora repo) |
|
|
|
''' |
|
export CUDA_VISIBLE_DEVICES=0,1 |
|
python train.py \ |
|
--train_type bnb_dora \ |
|
--model_name sosoai/hansoldeco-beomi-Llama-3-Open-Ko-8B-Instruct-preview \ |
|
--dataset orca_math \ |
|
--dataset_samples 193789 \ |
|
--batch_size 4 \ |
|
--context_length 8192 \ |
|
--gradient_accumulation_steps 2 \ |
|
--sharding_strategy full_shard \ |
|
--use_gradient_checkpointing true \ |
|
--reentrant_checkpointing true \ |
|
--use_cpu_offload false \ |
|
--use_activation_cpu_offload false \ |
|
--log_to wandb \ |
|
--project_name "sosoai-fsdp-quantized-ft-exps" \ |
|
--save_model true \ |
|
--output_dir models/llama-8b-orca-math-10k-bnb-QDoRA |
|
''' |
|
|
|
Dataset = hansoldeco domain own dataset (Non open) |
|
Dataset = kuotient/orca-math-word-problems-193k-korean |
|
|
|
|