{ | |
"include_wudao_corpus": true, | |
"lr": 1.5e-4, | |
"batch_size": 16, | |
"grad_accumulation_steps": 1, | |
"pad_token_id": 0, | |
"max_length": 896, | |
"epochs": 1, | |
"weight_decay": 0.1, | |
"interval": 2000, | |
"torch_dtype": null, | |
"data_path": "../data/*.bin", | |
"config_path": "../config/MiniLLM-1.1B-WithWudao/bert4torch_config.json", | |
"save_dir": "../ckpt/MiniLLM-1.1B-WithWudao" | |
} |