export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
torchrun --nproc-per-node 8 -m open_lm.main   \
 --model open_lm_1b \
 --workers 1 \
 --dataset-resampled \
 --precision amp_bfloat16 \
 --grad-checkpointing \
 --log-every-n-steps 100 \
 --train-num-samples 1000 \
 --grad-clip-norm 1 \
 --data-key jsonl \
 --dataset-type synthetic \
 --lr 1e-5 \
 --fsdp --fsdp-amp \
 --warmup 2000 \
 --wd 0.1 \
 --beta2 0.95 \
 --epochs 2 \
 --report-to tensorboard \
 --name open_lm_update_test_01 \