| nice nohup bash -c 'echo "start $(date)" && mpirun -np 8 ./train_gpt2cu \ | |
| -i "dev/data/fineweb10B/fineweb_train_*.bin" \ | |
| -j "dev/data/fineweb10B/fineweb_val_*.bin" \ | |
| -o log124M \ | |
| -e "d12" \ | |
| -b 64 -t 1024 \ | |
| -d 524288 \ | |
| -r 1 \ | |
| -z 1 \ | |
| -c 0.1 \ | |
| -l 0.0006 \ | |
| -q 0.0 \ | |
| -u 700 \ | |
| -n 5000 \ | |
| -y 1 \ | |
| -v 250 -s 20000 \ | |
| -h 1 && echo "end $(date)"' & | |