File size: 1,405 Bytes
f227cda |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
# GPT-3 (124M) repro on FineWeb
# 124M parameter model on 300B tokens
# note context length: 1024 -> 2048 for GPT-3
# => 6 * 124e6 * 300e9 = 7.44e18 ~= 2.2e20 capability model
# 565,950 steps of 524,288 tokens/step
# on 8X A100 80GB SXM ($14/hr) steps in ~300ms/iter
# => training time 565,950 * 300ms ~= 47 hours ~= $658
make train_gpt2cu USE_CUDNN=1
out_dir="log_gpt2_350M_edu_hermes"
done_file="$out_dir/DONE_00019622"
while true; do
# exit condition is that optimization has finished
if [ -f "$done_file" ]; then
echo "File $done_file exists. Exiting the loop."
break
fi
# run python dev/data/fineweb_edu_hermes.py --version 10B to prepro data
# run python dev/data/hellaswag.py to prepro hellaswag eval
mpirun -np 2 ./train_gpt2cu \
-i "dev/data/edu_fineweb10B_hermes/edu_fineweb_hermes_train_*.bin" \
-j "dev/data/edu_fineweb10B_hermes/edu_fineweb_hermes_val_*.bin" \
-o $out_dir \
-v 250 -s 5000 -g 144 \
-h 1 \
-b 16 -t 1024 \
-d 524288 \
-r 0 \
-z 1 \
-c 0.1 \
-l 0.0003 \
-q 0.0 \
-u 700 \
-n 5000 \
-sl 7.0 -sg 7.0 \
-y 1 \
-x 19622 \
-e "d24"
sleep 1
done
|