File size: 2,226 Bytes
de13652 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
{
"base_model_id": "deepseek-ai/deepseek-coder-1.3b-instruct",
"quantitize": "fp16",
"dataset": "Arithmetic_Hard",
"data_collator": "DataCollatorForLanguageModeling",
"peft_config": {
"lora": {
"r": 32,
"lora_alpha": 64,
"target_modules": [
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj"
],
"bias": "none",
"lora_dropout": 0.05,
"task_type": "CAUSAL_LM"
},
"lora_large": {
"r": 128,
"lora_alpha": 256,
"target_modules": [
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj"
],
"bias": "none",
"lora_dropout": 0.05,
"task_type": "CAUSAL_LM"
},
"p_tuning": {
"num_virtual_tokens": 16,
"num_transformer_submodules": 1,
"token_dim": 2048,
"encoder_hidden_size": 2048,
"task_type": "CAUSAL_LM"
}
},
"training_args": {
"warmup_steps": 500,
"per_device_train_batch_size": 4,
"per_device_eval_batch_size": 4,
"gradient_accumulation_steps": 1,
"max_steps": 100000,
"learning_rate": 0.0001,
"optim": "paged_adamw_8bit",
"logging_steps": 100,
"save_strategy": "steps",
"save_total_limit": 5,
"save_steps": 2500,
"evaluation_strategy": "steps",
"eval_steps": 2500,
"weight_decay": 0.01,
"report_to": "wandb",
"dataloader_num_workers": 4,
"load_best_model_at_end": true,
"fp16": false,
"output_dir": "runs/deepseek-full-hard",
"logging_dir": "runs/deepseek-full-hard/logs"
},
"tokenizer": {
"tokenize_config": {
"truncation": true,
"max_length": 512,
"padding": "max_length"
},
"prompt_template": "config/qa_template.txt"
}
}
|