[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-grpo-fsdp/run_qwen3-0.6b-grpo-fsdp_36234.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-grpo-fsdp/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-grpo-fsdp/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-grpo-fsdp

cat $0

source ~/.bashrc

micromamba activate miles

sh scripts/run-qwen3-0.6B-fsdp.shRunning inside SLURM, Job ID 36234.
2025-12-10 14:29:47,972	INFO scripts.py:1371 -- Did not find any active Ray processes.
pkill: killing pid 1700948 failed: Operation not permitted
pkill: killing pid 1702340 failed: Operation not permitted
pkill: killing pid 1703925 failed: Operation not permitted
pkill: killing pid 1700948 failed: Operation not permitted
pkill: killing pid 1702340 failed: Operation not permitted
pkill: killing pid 1703925 failed: Operation not permitted
+ export PYTHONUNBUFFERED=1
+ PYTHONUNBUFFERED=1
+ export CUDA_VISIBLE_DEVICES=0,1
+ CUDA_VISIBLE_DEVICES=0,1
++ nvidia-smi
++ grep -o NVLink
++ wc -l
+ NVLINK_COUNT=0
+ '[' 0 -gt 0 ']'
+ HAS_NVLINK=0
+ echo 'HAS_NVLINK: 0 (detected 0 NVLink references)'
HAS_NVLINK: 0 (detected 0 NVLink references)
++ hostname -I
++ awk '{print $1}'
+ export HEAD_NODE_IP=10.86.2.80
+ HEAD_NODE_IP=10.86.2.80
+ echo 'Detected Head Node IP: 10.86.2.80'
Detected Head Node IP: 10.86.2.80
+ export no_proxy=10.86.2.80,localhost,127.0.0.1,0.0.0.0
+ no_proxy=10.86.2.80,localhost,127.0.0.1,0.0.0.0
+ export NO_PROXY=10.86.2.80,localhost,127.0.0.1,0.0.0.0
+ NO_PROXY=10.86.2.80,localhost,127.0.0.1,0.0.0.0
+++ dirname -- scripts/run-qwen3-0.6B-fsdp.sh
++ cd -- scripts
++ pwd
+ SCRIPT_DIR=/home/mihir.mahajan/Projects/miles/scripts
++ date +%Y%m%d_%H%M%S
+ RUN_ID=run_20251210_142951
+ LOAD_SAVE_PATH=/fast/project/HFMI_SynergyUnit/tab_model/huggingface/shared_data/run_20251210_142951/checkpoints
+ CKPT_ARGS=(--hf-checkpoint /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --ref-load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B)
+ ROLLOUT_ARGS=(--prompt-data /fast/project/HFMI_SynergyUnit/tab_model/huggingface/dapo-math-17k/dapo-math-17k.jsonl --input-key prompt --label-key label --apply-chat-template --rollout-shuffle --balance-data --rm-type deepscaler --num-rollout 100 --rollout-batch-size 4 --n-samples-per-prompt 4 --rollout-max-response-len 4096 --rollout-temperature 0.8 --global-batch-size 64)
+ GRPO_ARGS=(--use-kl-loss --advantage-estimator grpo --kl-loss-coef 0.00 --kl-loss-type low_var_kl --kl-coef 0.00 --entropy-coef 0.00 --eps-clip 0.2 --eps-clip-high 0.28)
+ OPTIMIZER_ARGS=(--optimizer adam --lr 1e-6 --lr-decay-style constant --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.98)
+ WANDB_ARGS=(--use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen3-0.6b-grpo-fsdp)
+ SGLANG_ARGS=(--rollout-num-gpus-per-engine 2 --sglang-mem-fraction-static 0.5 --sglang-decode-log-interval 1000 --sglang-chunked-prefill-size 4096 --sglang-attention-backend fa3)
+ TRAIN_BACKEND_ARGS=(--train-backend fsdp --update-weight-buffer-size 536870912 --gradient-checkpointing --attn-implementation flash_attention_3 --train-env-vars '{"PYTORCH_CUDA_ALLOC_CONF":"expandable_segments:True"}')
+ PERF_ARGS=(--use-dynamic-batch-size --max-tokens-per-gpu 9216)
+ MISC_ARGS=(--actor-num-nodes 1 --actor-num-gpus-per-node 2 --colocate --use-fault-tolerance --dump-details /fast/project/HFMI_SynergyUnit/tab_model/huggingface/shared_data/qwen3-600M-fsdp-1116-noref/dump_details)
+ export MASTER_ADDR=127.0.0.1
+ MASTER_ADDR=127.0.0.1
+ export PYTHONPATH=/fast/project/HFMI_SynergyUnit/mihir/Megatron-LM/:/home/mihir.mahajan/Projects/miles/scripts
+ PYTHONPATH=/fast/project/HFMI_SynergyUnit/mihir/Megatron-LM/:/home/mihir.mahajan/Projects/miles/scripts
+ python3 -m ray.scripts.scripts start --head --node-ip-address=10.86.2.80 --num-gpus 2 --num-cpus 8 --disable-usage-stats --dashboard-host=0.0.0.0 --dashboard-port=8265 --port=6379
2025-12-10 14:29:54,689	INFO usage_lib.py:447 -- Usage stats collection is disabled.
2025-12-10 14:29:54,691	INFO scripts.py:919 -- Local node IP: 10.86.2.80