[tab] /p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/qwen_1.7b_no_lora_bs_16_13068219.log (log)
#!/usr/bin/env bash
#
# SBATCH Template Script for SFT Training
# This script is invoked by the launcher with environment variables set
#
# Required environment variables (set by launcher):
#   - MODEL_SIZE: 1.7b, 4b, or 8b
#   - BATCH_SIZE: 8, 16, or 32
#   - USE_LORA: true or false
#

# #SBATCH --nodes=4  # Set dynamically by launcher
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --partition=booster
#SBATCH --gres=gpu:4
#SBATCH --mem=200GB
#SBATCH --account=envcomp
#SBATCH --nodelist=jwb[0700-0799]
#SBATCH --signal=USR1@60
#SBATCH --output=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log

trap 'echo "Job killed by signal"' USR1

# Print this script for debugging
cat $0

echo "============================================"
echo "SBATCH Environment:"
echo "  MODEL_SIZE: ${MODEL_SIZE}"
echo "  BATCH_SIZE: ${BATCH_SIZE}"
echo "  USE_LORA:   ${USE_LORA}"
echo "  CP_SIZE:    ${CONTEXT_PARALLEL_SIZE}"
echo "  SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "============================================"

# Load required modules
module load Stages/2025
module load CUDA/12
module load cuDNN/9.5.0.50-CUDA-12
module load NCCL/default-CUDA-12
module load Clang/18.1.8

# Activate virtual environment
source .venv/bin/activate

# Determine Master Address once to avoid multiple scontrol calls
export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
export MASTER_PORT=29500
echo "============================================"
echo "Master node is ${MASTER_ADDR} on port ${MASTER_PORT}"
echo "============================================"

# Export configuration for the run script
export MODEL_SIZE="${MODEL_SIZE}"
export BATCH_SIZE="${BATCH_SIZE}"
export USE_LORA="${USE_LORA}"
export CONTEXT_PARALLEL_SIZE="${CONTEXT_PARALLEL_SIZE}"

echo "Launching training script on all nodes via srun..."
# Run the training script on all nodes
srun sh slurm/jobs/mihir/juelich/crowd-pilot/2-wave-broken-resolutions/run_template.sh
echo "srun execution finished."
============================================
SBATCH Environment:
  MODEL_SIZE: 1.7b
  BATCH_SIZE: 16
  USE_LORA:   false
  CP_SIZE:    1
  SLURM_JOB_ID: 13068219
============================================

  This stage is deprecated. Please consider moving to a new stage (2026
or newer)


Due to MODULEPATH changes, the following have been reloaded:
  1) make/4.4.1

The following have been reloaded with a version change:
  1) GCCcore/.14.3.0 => GCCcore/.13.3.0     4) binutils/2.44 => binutils/.2.42
  2) Stages/2026 => Stages/2025             5) zlib/1.3.1 => zlib/.1.3.1
  3) StdEnv/2026 => StdEnv/2025

============================================
Master node is jwb0763 on port 29500
============================================
Launching training script on all nodes via srun...
Environment variables have been set
Environment variables have been set
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-1.7B
============================================
Run Configuration:
  MODEL_SIZE:  1.7b
  MODEL_NAME:  Qwen3-1.7B
  BATCH_SIZE:  16
  USE_LORA:    false
  CP_SIZE:     1
  VAL_INT:     125
  NUM_ROLL:    5000
  LR_WARMUP:   250
  RUN_NAME:    qwen1.7B_no_lora_bs_16_13068219
  RUN_ID:      13068219
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068219/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen1.7B_no_lora_bs_16_13068219 --wandb-run-id 13068219 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-1.7B
============================================
Run Configuration:
  MODEL_SIZE:  1.7b
  MODEL_NAME:  Qwen3-1.7B
  BATCH_SIZE:  16
  USE_LORA:    false
  CP_SIZE:     1
  VAL_INT:     125
  NUM_ROLL:    5000
  LR_WARMUP:   250
  RUN_NAME:    qwen1.7B_no_lora_bs_16_13068219
  RUN_ID:      13068219
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068219/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen1.7B_no_lora_bs_16_13068219 --wandb-run-id 13068219 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
W0107 22:46:20.443000 1438992 torch/distributed/run.py:774] 
W0107 22:46:20.443000 1438992 torch/distributed/run.py:774] *****************************************
W0107 22:46:20.443000 1438992 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 22:46:20.443000 1438992 torch/distributed/run.py:774] *****************************************
[W107 22:46:20.614026423 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0763.juwels]:29500 (errno: 97 - Address family not supported by protocol).
W0107 22:46:20.477000 1048561 torch/distributed/run.py:774] 
W0107 22:46:20.477000 1048561 torch/distributed/run.py:774] *****************************************
W0107 22:46:20.477000 1048561 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 22:46:20.477000 1048561 torch/distributed/run.py:774] *****************************************
[W107 22:46:20.678675610 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0763.juwels]:29500 (errno: 97 - Address family not supported by protocol).
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
[2026-01-07 22:47:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:47:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:47:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:47:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:47:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:47:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:47:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:47:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:47:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:47:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:47:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:47:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:47:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:47:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:47:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:47:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[W107 22:47:34.261507287 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0763.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:47:34.262224453 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:47:34.263114641 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0763.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:47:34.263540885 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:47:34.268352329 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0763.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:47:34.268698412 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:47:34.273626787 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0763.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:47:34.274013721 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:47:34.269358223 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0763.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:47:34.270022157 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0763.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:47:34.270356789 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:47:34.270788372 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:47:34.272757874 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0763.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:47:34.273376698 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:47:34.278193247 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0763.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:47:34.278897711 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[2026-01-07 22:47:34] train_sft.py:120 - [Rank 0] Distributed initialized: world_size=8, local_rank=0
[2026-01-07 22:47:34] train_sft.py:120 - [Rank 1] Distributed initialized: world_size=8, local_rank=1
[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to  peer ranks. 7Expected number of connected peer ranks is :  peer ranks. 7Expected number of connected peer ranks is : 7

[2026-01-07 22:47:34] train_sft.py:120 - [Rank 3] Distributed initialized: world_size=8, local_rank=3
[2026-01-07 22:47:34] train_sft.py:120 - [Rank 2] Distributed initialized: world_size=8, local_rank=2
[2026-01-07 22:47:34] train_sft.py:120 - [Rank 5] Distributed initialized: world_size=8, local_rank=1
[2026-01-07 22:47:34] train_sft.py:120 - [Rank 7] Distributed initialized: world_size=8, local_rank=3
[2026-01-07 22:47:34] train_sft.py:120 - [Rank 4] Distributed initialized: world_size=8, local_rank=0
[2026-01-07 22:47:34] train_sft.py:120 - [Rank 6] Distributed initialized: world_size=8, local_rank=2
[2026-01-07 22:47:34] train_sft.py:146 - [Rank 1] Device mesh: dp_size=8, cp_size=1, dp_rank=1, cp_rank=0
[2026-01-07 22:47:34] train_sft.py:146 - [Rank 2] Device mesh: dp_size=8, cp_size=1, dp_rank=2, cp_rank=0
[2026-01-07 22:47:34] train_sft.py:146 - [Rank 0] Device mesh: dp_size=8, cp_size=1, dp_rank=0, cp_rank=0
[2026-01-07 22:47:34] train_sft.py:146 - [Rank 3] Device mesh: dp_size=8, cp_size=1, dp_rank=3, cp_rank=0
[2026-01-07 22:47:34] train_sft.py:146 - [Rank 7] Device mesh: dp_size=8, cp_size=1, dp_rank=7, cp_rank=0
[2026-01-07 22:47:34] train_sft.py:146 - [Rank 4] Device mesh: dp_size=8, cp_size=1, dp_rank=4, cp_rank=0
[2026-01-07 22:47:34] train_sft.py:146 - [Rank 6] Device mesh: dp_size=8, cp_size=1, dp_rank=6, cp_rank=0
[2026-01-07 22:47:34] train_sft.py:146 - [Rank 5] Device mesh: dp_size=8, cp_size=1, dp_rank=5, cp_rank=0
[2026-01-07 22:47:34] wandb_utils.py:72 - W&B logs will be stored in: /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
wandb: Tracking run with wandb version 0.23.1
wandb: W&B syncing is set to `offline` in this directory. Run `wandb online` or set WANDB_MODE=online to enable cloud syncing.
wandb: Run data is saved locally in /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS/wandb/offline-run-20260107_224734-bz1xyo9s
wandb: Detected [openai] in use.
wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
[2026-01-07 22:49:11] train_sft.py:218 - [Rank 3] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s][2026-01-07 22:49:11] train_sft.py:218 - [Rank 2] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s][2026-01-07 22:49:12] train_sft.py:218 - [Rank 6] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s][2026-01-07 22:49:12] train_sft.py:218 - [Rank 1] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s][2026-01-07 22:49:13] train_sft.py:218 - [Rank 4] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s][2026-01-07 22:49:13] train_sft.py:218 - [Rank 0] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s][2026-01-07 22:49:13] train_sft.py:218 - [Rank 7] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 22:49:13] train_sft.py:218 - [Rank 5] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.81s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.45s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:10<00:10, 10.83s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:10<00:10, 10.99s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:10<00:10, 10.16s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.55s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:09<00:09,  9.52s/it]
Loading checkpoint shards:  50%|█████     | 1/2 [00:10<00:10, 10.61s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:12<00:00,  5.50s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:12<00:00,  6.32s/it]

Loading checkpoint shards: 100%|██████████| 2/2 [00:11<00:00,  5.04s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:11<00:00,  5.76s/it]

Loading checkpoint shards: 100%|██████████| 2/2 [00:11<00:00,  4.87s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:11<00:00,  5.56s/it]

Loading checkpoint shards: 100%|██████████| 2/2 [00:12<00:00,  5.45s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:12<00:00,  6.25s/it]
[2026-01-07 22:49:24] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:49:24] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:49:24] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:49:24] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32

Loading checkpoint shards: 100%|██████████| 2/2 [00:12<00:00,  5.45s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:12<00:00,  6.23s/it]

Loading checkpoint shards: 100%|██████████| 2/2 [00:11<00:00,  5.01s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:11<00:00,  5.03s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:11<00:00,  5.69s/it]

Loading checkpoint shards: 100%|██████████| 2/2 [00:11<00:00,  5.70s/it]

Loading checkpoint shards: 100%|██████████| 2/2 [00:12<00:00,  5.30s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:12<00:00,  6.03s/it]
[2026-01-07 22:49:25] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:49:25] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:49:25] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:49:25] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:49:28] train_sft.py:321 - [Rank 5] Model initialized with FSDP
[2026-01-07 22:49:28] train_sft.py:321 - [Rank 7] Model initialized with FSDP
[2026-01-07 22:49:28] train_sft.py:321 - [Rank 4] Model initialized with FSDP
[2026-01-07 22:49:28] train_sft.py:321 - [Rank 6] Model initialized with FSDP
[2026-01-07 22:49:28] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:49:28] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:49:28] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:49:28] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 22:49:28] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 22:49:28] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 22:49:28] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:49:28] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:49:28] train_sft.py:321 - [Rank 2] Model initialized with FSDP
[2026-01-07 22:49:28] train_sft.py:321 - [Rank 3] Model initialized with FSDP
[2026-01-07 22:49:28] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:49:28] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:49:28] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 22:49:28] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:49:28] train_sft.py:321 - [Rank 1] Model initialized with FSDP
[2026-01-07 22:49:28] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:49:28] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:49:28] train_sft.py:321 - [Rank 0] Model initialized with FSDP
[2026-01-07 22:49:28] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:49:28] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:49:28] train_sft.py:102 - [Rank 1] SFTTrainer initialized successfully
[2026-01-07 22:49:28] train_sft.py:102 - [Rank 0] SFTTrainer initialized successfully
[2026-01-07 22:49:28] train_sft.py:102 - [Rank 2] SFTTrainer initialized successfully
[2026-01-07 22:49:28] train_sft.py:102 - [Rank 3] SFTTrainer initialized successfully
[2026-01-07 22:49:28] train_sft.py:701 - [Rank 2] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:49:28] train_sft.py:701 - [Rank 1] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:49:28] train_sft.py:701 - [Rank 3] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:49:28] train_sft.py:701 - [Rank 0] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:49:28] train_sft.py:102 - [Rank 5] SFTTrainer initialized successfully
[2026-01-07 22:49:28] train_sft.py:102 - [Rank 7] SFTTrainer initialized successfully
[2026-01-07 22:49:28] train_sft.py:102 - [Rank 6] SFTTrainer initialized successfully
[2026-01-07 22:49:28] train_sft.py:102 - [Rank 4] SFTTrainer initialized successfully
[2026-01-07 22:49:28] train_sft.py:701 - [Rank 7] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:49:28] train_sft.py:701 - [Rank 5] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:49:28] train_sft.py:701 - [Rank 6] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:49:28] train_sft.py:701 - [Rank 4] Starting training: rollout_id 0 -> 5000

actor_val:   0%|          | 0/100 [00:00<?, ?it/s][2026-01-07 22:49:28] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:   1%|          | 1/100 [00:04<07:50,  4.75s/it][2026-01-07 22:49:33] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:   2%|▏         | 2/100 [00:05<03:45,  2.30s/it][2026-01-07 22:49:34] train_sft.py:375 - SFT rollout sample: tokens_len=3530, response_length=2945

actor_val:   3%|▎         | 3/100 [00:05<02:27,  1.52s/it][2026-01-07 22:49:34] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960
[2026-01-07 22:49:34] train_sft.py:637 - [Rank 7] No batches to validate on rollout 0, validation step 3
[2026-01-07 22:49:34] train_sft.py:637 - [Rank 6] No batches to validate on rollout 0, validation step 3
[2026-01-07 22:49:34] train_sft.py:637 - [Rank 4] No batches to validate on rollout 0, validation step 3
[2026-01-07 22:49:34] train_sft.py:637 - [Rank 5] No batches to validate on rollout 0, validation step 3
srun: Job step aborted: Waiting up to 6 seconds for job step to finish.
W0108 13:52:02.935000 1048561 torch/distributed/elastic/agent/server/api.py:723] Received 15 death signal, shutting down workers
W0108 13:52:02.956000 1048561 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 1048608 closing signal SIGTERM
W0108 13:52:02.958000 1048561 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 1048609 closing signal SIGTERM
W0108 13:52:02.959000 1048561 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 1048610 closing signal SIGTERM
W0108 13:52:02.960000 1048561 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 1048611 closing signal SIGTERM
Traceback (most recent call last):
  File "/p/project1/envcomp/mahajan1/miles/.venv/bin/torchrun", line 10, in <module>
    sys.exit(main())
             ^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
    return f(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/run.py", line 901, in main
    run(args)
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/run.py", line 892, in run
    elastic_launch(
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 143, in __call__
    return launch_agent(self._config, self._entrypoint, list(args))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 268, in launch_agent
    result = agent.run()
             ^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
    result = f(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/agent/server/api.py", line 715, in run
    result = self._invoke_run(role)
             ^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/agent/server/api.py", line 879, in _invoke_run
    time.sleep(monitor_interval)
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
torch.distributed.elastic.multiprocessing.api.SignalException: Process 1048561 got signal: 15
srun: error: jwb0763: task 0: Terminated
srun execution finished.