[tab] slurm/utils/mihir/weekend-job-requeuer.sh (shellscript)

echo "[INFO] weekend requeuer started at $(date)"
sleep 9h
echo "[INFO] 9 hours passed at $(date)"

# actionspace scaling
echo "[INFO] Requeueing actionspace scaling jobs at $(date)"
echo "[INFO] Submitting: train_dynamics_cotraining_12_actions.sbatch"
sbatch slurm/jobs/mihir/horeka/action_space_scaling/co_training/train_dynamics_cotraining_12_actions.sbatch
echo "[INFO] Submitting: train_dynamics_cotraining_20_actions.sbatch"
sbatch slurm/jobs/mihir/horeka/action_space_scaling/co_training/train_dynamics_cotraining_20_actions.sbatch
echo "[INFO] Submitting: train_dynamics_cotraining_50_actions.sbatch"
sbatch slurm/jobs/mihir/horeka/action_space_scaling/co_training/train_dynamics_cotraining_50_actions.sbatch
echo "[INFO] Submitting: train_dynamics_cotraining_6_actions.sbatch"
sbatch slurm/jobs/mihir/horeka/action_space_scaling/co_training/train_dynamics_cotraining_6_actions.sbatch
echo "[INFO] Submitting: train_dynamics_cotraining_8_actions.sbatch"
sbatch slurm/jobs/mihir/horeka/action_space_scaling/co_training/train_dynamics_cotraining_8_actions.sbatch

# lr sweep
sleep 1h
echo "[INFO] Requeueing lr sweep jobs at $(date)"
echo "[INFO] Submitting: train_tokenizer_lr_1e-4.sbatch"
sbatch slurm/jobs/mihir/horeka/lr_tuning/tokenizer/train_tokenizer_lr_1e-4.sbatch
echo "[INFO] Submitting: train_tokenizer_lr_5e-5.sbatch"
sbatch slurm/jobs/mihir/horeka/lr_tuning/tokenizer/train_tokenizer_lr_5e-5.sbatch

# batchsize scaling
sleep 1h
echo "[INFO] Requeueing batchsize scaling runs: [1,2]nodes at $(date)"
echo "[INFO] Submitting: train_dynamics_1_nodes.sbatch"
sbatch slurm/jobs/mihir/horeka/batchsize_scaling/dynamics_cotraining/sqrt_lr/train_dynamics_1_nodes.sbatch
echo "[INFO] Submitting: train_dynamics_2_nodes.sbatch"
sbatch slurm/jobs/mihir/horeka/batchsize_scaling/dynamics_cotraining/sqrt_lr/train_dynamics_2_nodes.sbatch

sleep 3h
echo "[INFO] Requeueing batchsize scaling runs: [4,16]nodes at $(date)"
echo "[INFO] Submitting: train_dynamics_16_nodes.sbatch"
sbatch slurm/jobs/mihir/horeka/batchsize_scaling/dynamics_cotraining/sqrt_lr/train_dynamics_16_nodes.sbatch
echo "[INFO] Submitting: train_dynamics_4_nodes.sbatch"
sbatch slurm/jobs/mihir/horeka/batchsize_scaling/dynamics_cotraining/sqrt_lr/train_dynamics_4_nodes.sbatch

sleep 6h
echo "[INFO] Requeueing batchsize scaling runs: 8nodes at $(date)"
echo "[INFO] Submitting: train_dynamics_8_nodes.sbatch"
sbatch slurm/jobs/mihir/horeka/batchsize_scaling/dynamics_cotraining/sqrt_lr/train_dynamics_8_nodes.sbatch

# modelsize scaling
sleep 2h
echo "[INFO] Requeueing modelsize scaling runs at $(date)"
echo "[INFO] Submitting: 1_train_dyn_36M.sbatch"
sbatch slurm/jobs/mihir/horeka/modelsize_scaling/dynamics/1_train_dyn_36M.sbatch
echo "[INFO] Submitting: 2_train_dyn_110M.sbatch"
sbatch slurm/jobs/mihir/horeka/modelsize_scaling/dynamics/2_train_dyn_110M.sbatch
echo "[INFO] Submitting: 3_train_dyn_180M.sbatch"
sbatch slurm/jobs/mihir/horeka/modelsize_scaling/dynamics/3_train_dyn_180M.sbatch
# echo "[INFO] Submitting: 4_train_dyn_270M.sbatch"
# sbatch slurm/jobs/mihir/horeka/modelsize_scaling/dynamics/4_train_dyn_270M.sbatch
# echo "[INFO] Submitting: 5_train_dyn_500M.sbatch"
# sbatch slurm/jobs/mihir/horeka/modelsize_scaling/dynamics/5_train_dyn_500M.sbatch