[tab] /p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/qwen_4b_no_lora_bs_8_13068223.log (log)
#!/usr/bin/env bash
#
# SBATCH Template Script for SFT Training
# This script is invoked by the launcher with environment variables set
#
# Required environment variables (set by launcher):
#   - MODEL_SIZE: 1.7b, 4b, or 8b
#   - BATCH_SIZE: 8, 16, or 32
#   - USE_LORA: true or false
#

# #SBATCH --nodes=4  # Set dynamically by launcher
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --partition=booster
#SBATCH --gres=gpu:4
#SBATCH --mem=200GB
#SBATCH --account=envcomp
#SBATCH --nodelist=jwb[0700-0799]
#SBATCH --signal=USR1@60
#SBATCH --output=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log

trap 'echo "Job killed by signal"' USR1

# Print this script for debugging
cat $0

echo "============================================"
echo "SBATCH Environment:"
echo "  MODEL_SIZE: ${MODEL_SIZE}"
echo "  BATCH_SIZE: ${BATCH_SIZE}"
echo "  USE_LORA:   ${USE_LORA}"
echo "  CP_SIZE:    ${CONTEXT_PARALLEL_SIZE}"
echo "  SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "============================================"

# Load required modules
module load Stages/2025
module load CUDA/12
module load cuDNN/9.5.0.50-CUDA-12
module load NCCL/default-CUDA-12
module load Clang/18.1.8

# Activate virtual environment
source .venv/bin/activate

# Determine Master Address once to avoid multiple scontrol calls
export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
export MASTER_PORT=29500
echo "============================================"
echo "Master node is ${MASTER_ADDR} on port ${MASTER_PORT}"
echo "============================================"

# Export configuration for the run script
export MODEL_SIZE="${MODEL_SIZE}"
export BATCH_SIZE="${BATCH_SIZE}"
export USE_LORA="${USE_LORA}"
export CONTEXT_PARALLEL_SIZE="${CONTEXT_PARALLEL_SIZE}"

echo "Launching training script on all nodes via srun..."
# Run the training script on all nodes
srun sh slurm/jobs/mihir/juelich/crowd-pilot/2-wave-broken-resolutions/run_template.sh
echo "srun execution finished."
============================================
SBATCH Environment:
  MODEL_SIZE: 4b
  BATCH_SIZE: 8
  USE_LORA:   false
  CP_SIZE:    1
  SLURM_JOB_ID: 13068223
============================================

  This stage is deprecated. Please consider moving to a new stage (2026
or newer)


Due to MODULEPATH changes, the following have been reloaded:
  1) make/4.4.1

The following have been reloaded with a version change:
  1) GCCcore/.14.3.0 => GCCcore/.13.3.0     4) binutils/2.44 => binutils/.2.42
  2) Stages/2026 => Stages/2025             5) zlib/1.3.1 => zlib/.1.3.1
  3) StdEnv/2026 => StdEnv/2025

============================================
Master node is jwb0757 on port 29500
============================================
Launching training script on all nodes via srun...
Environment variables have been set
Environment variables have been set
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  8
  USE_LORA:    false
  CP_SIZE:     1
  VAL_INT:     250
  NUM_ROLL:    10000
  LR_WARMUP:   500
  RUN_NAME:    qwen4B_no_lora_bs_8_13068223
  RUN_ID:      13068223
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068223/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_no_lora_bs_8_13068223 --wandb-run-id 13068223 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  8
  USE_LORA:    false
  CP_SIZE:     1
  VAL_INT:     250
  NUM_ROLL:    10000
  LR_WARMUP:   500
  RUN_NAME:    qwen4B_no_lora_bs_8_13068223
  RUN_ID:      13068223
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068223/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_no_lora_bs_8_13068223 --wandb-run-id 13068223 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
W0108 00:27:06.529000 622874 torch/distributed/run.py:774] 
W0108 00:27:06.529000 622874 torch/distributed/run.py:774] *****************************************
W0108 00:27:06.529000 622874 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0108 00:27:06.529000 622874 torch/distributed/run.py:774] *****************************************
[W108 00:27:06.848159470 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0757.juwels]:29500 (errno: 97 - Address family not supported by protocol).
W0108 00:27:07.530000 254659 torch/distributed/run.py:774] 
W0108 00:27:07.530000 254659 torch/distributed/run.py:774] *****************************************
W0108 00:27:07.530000 254659 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0108 00:27:07.530000 254659 torch/distributed/run.py:774] *****************************************
[W108 00:27:07.689507919 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0757.juwels]:29500 (errno: 97 - Address family not supported by protocol).
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
[2026-01-08 00:28:31] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-08 00:28:31] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-08 00:28:31] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-08 00:28:31] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-08 00:28:31] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-08 00:28:31] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-08 00:28:31] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-08 00:28:31] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-08 00:28:31] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-08 00:28:31] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-08 00:28:31] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-08 00:28:31] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-08 00:28:31] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-08 00:28:31] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-08 00:28:31] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-08 00:28:31] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[W108 00:28:31.181404418 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0757.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W108 00:28:31.182205376 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W108 00:28:31.193206660 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0757.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W108 00:28:31.193206310 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0757.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W108 00:28:31.193610054 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W108 00:28:31.193646355 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W108 00:28:32.213082569 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0757.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W108 00:28:32.213487393 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W108 00:28:32.224097120 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0757.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W108 00:28:32.224948109 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W108 00:28:32.228243577 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0757.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W108 00:28:32.228931446 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W108 00:28:32.238898237 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0757.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W108 00:28:32.239595006 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W108 00:28:32.246429230 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0757.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W108 00:28:32.247054370 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[2026-01-08 00:28:32] train_sft.py:120 - [Rank 4] Distributed initialized: world_size=8, local_rank=0
[2026-01-08 00:28:32] train_sft.py:120 - [Rank 5] Distributed initialized: world_size=8, local_rank=1
[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[2026-01-08 00:28:32] train_sft.py:120 - [Rank 0] Distributed initialized: world_size=8, local_rank=0
[2026-01-08 00:28:32] train_sft.py:120 - [Rank 1] Distributed initialized: world_size=8, local_rank=1
[2026-01-08 00:28:32] train_sft.py:120 - [Rank 2] Distributed initialized: world_size=8, local_rank=2
[2026-01-08 00:28:32] train_sft.py:120 - [Rank 7] Distributed initialized: world_size=8, local_rank=3
[2026-01-08 00:28:32] train_sft.py:120 - [Rank 3] Distributed initialized: world_size=8, local_rank=3
[2026-01-08 00:28:32] train_sft.py:120 - [Rank 6] Distributed initialized: world_size=8, local_rank=2
[2026-01-08 00:28:32] train_sft.py:146 - [Rank 5] Device mesh: dp_size=8, cp_size=1, dp_rank=5, cp_rank=0
[2026-01-08 00:28:32] train_sft.py:146 - [Rank 4] Device mesh: dp_size=8, cp_size=1, dp_rank=4, cp_rank=0
[2026-01-08 00:28:32] train_sft.py:146 - [Rank 6] Device mesh: dp_size=8, cp_size=1, dp_rank=6, cp_rank=0
[2026-01-08 00:28:32] train_sft.py:146 - [Rank 7] Device mesh: dp_size=8, cp_size=1, dp_rank=7, cp_rank=0
[2026-01-08 00:28:32] train_sft.py:146 - [Rank 3] Device mesh: dp_size=8, cp_size=1, dp_rank=3, cp_rank=0
[2026-01-08 00:28:32] train_sft.py:146 - [Rank 1] Device mesh: dp_size=8, cp_size=1, dp_rank=1, cp_rank=0
[2026-01-08 00:28:32] train_sft.py:146 - [Rank 0] Device mesh: dp_size=8, cp_size=1, dp_rank=0, cp_rank=0
[2026-01-08 00:28:32] train_sft.py:146 - [Rank 2] Device mesh: dp_size=8, cp_size=1, dp_rank=2, cp_rank=0
[2026-01-08 00:28:32] wandb_utils.py:72 - W&B logs will be stored in: /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
wandb: Tracking run with wandb version 0.23.1
wandb: W&B syncing is set to `offline` in this directory. Run `wandb online` or set WANDB_MODE=online to enable cloud syncing.
wandb: Run data is saved locally in /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS/wandb/offline-run-20260108_002832-37m2paam
wandb: Detected [openai] in use.
wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
[2026-01-08 00:30:13] train_sft.py:218 - [Rank 7] Data source initialized: dataset_size=97, num_rollout=10000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-08 00:30:14] train_sft.py:218 - [Rank 4] Data source initialized: dataset_size=97, num_rollout=10000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-08 00:30:14] train_sft.py:218 - [Rank 0] Data source initialized: dataset_size=97, num_rollout=10000
[2026-01-08 00:30:14] train_sft.py:218 - [Rank 1] Data source initialized: dataset_size=97, num_rollout=10000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-08 00:30:15] train_sft.py:218 - [Rank 6] Data source initialized: dataset_size=97, num_rollout=10000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-08 00:30:15] train_sft.py:218 - [Rank 5] Data source initialized: dataset_size=97, num_rollout=10000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-08 00:30:15] train_sft.py:218 - [Rank 2] Data source initialized: dataset_size=97, num_rollout=10000

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-08 00:30:15] train_sft.py:218 - [Rank 3] Data source initialized: dataset_size=97, num_rollout=10000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:20, 10.39s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.86s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:20, 10.39s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.45s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.73s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:12<00:24, 12.19s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:12<00:24, 12.19s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.75s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:24<00:12, 12.45s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:13, 13.06s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.87s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:24<00:12, 12.50s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.21s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.59s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.31s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  6.99s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  8.26s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.74s/it]


Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  6.98s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  8.26s/it]
[2026-01-08 00:30:39] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-08 00:30:39] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-08 00:30:39] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-08 00:30:39] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32

Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.69s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.67s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.88s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.91s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.18s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.65s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.18s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.08s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.50s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.65s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.07s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.49s/it]
[2026-01-08 00:30:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-08 00:30:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-08 00:30:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-08 00:30:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-08 00:30:46] train_sft.py:321 - [Rank 2] Model initialized with FSDP
[2026-01-08 00:30:46] train_sft.py:321 - [Rank 3] Model initialized with FSDP
[2026-01-08 00:30:46] train_sft.py:321 - [Rank 1] Model initialized with FSDP
[2026-01-08 00:30:46] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-08 00:30:46] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-08 00:30:46] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-08 00:30:46] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-08 00:30:46] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-08 00:30:46] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-08 00:30:46] train_sft.py:321 - [Rank 7] Model initialized with FSDP
[2026-01-08 00:30:46] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-08 00:30:46] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-08 00:30:46] train_sft.py:321 - [Rank 5] Model initialized with FSDP
[2026-01-08 00:30:46] train_sft.py:321 - [Rank 4] Model initialized with FSDP
[2026-01-08 00:30:46] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-08 00:30:46] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-08 00:30:46] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-08 00:30:46] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-08 00:30:46] train_sft.py:321 - [Rank 6] Model initialized with FSDP
[2026-01-08 00:30:46] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-08 00:30:46] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-08 00:30:49] train_sft.py:321 - [Rank 0] Model initialized with FSDP
[2026-01-08 00:30:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-08 00:30:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-08 00:30:49] train_sft.py:102 - [Rank 1] SFTTrainer initialized successfully
[2026-01-08 00:30:49] train_sft.py:102 - [Rank 2] SFTTrainer initialized successfully
[2026-01-08 00:30:49] train_sft.py:102 - [Rank 0] SFTTrainer initialized successfully
[2026-01-08 00:30:49] train_sft.py:102 - [Rank 3] SFTTrainer initialized successfully
[2026-01-08 00:30:49] train_sft.py:701 - [Rank 1] Starting training: rollout_id 0 -> 10000
[2026-01-08 00:30:49] train_sft.py:701 - [Rank 2] Starting training: rollout_id 0 -> 10000
[2026-01-08 00:30:49] train_sft.py:701 - [Rank 3] Starting training: rollout_id 0 -> 10000
[2026-01-08 00:30:49] train_sft.py:701 - [Rank 0] Starting training: rollout_id 0 -> 10000
[2026-01-08 00:30:49] train_sft.py:102 - [Rank 4] SFTTrainer initialized successfully
[2026-01-08 00:30:49] train_sft.py:102 - [Rank 7] SFTTrainer initialized successfully
[2026-01-08 00:30:49] train_sft.py:102 - [Rank 6] SFTTrainer initialized successfully
[2026-01-08 00:30:49] train_sft.py:701 - [Rank 7] Starting training: rollout_id 0 -> 10000
[2026-01-08 00:30:49] train_sft.py:701 - [Rank 4] Starting training: rollout_id 0 -> 10000
[2026-01-08 00:30:49] train_sft.py:701 - [Rank 6] Starting training: rollout_id 0 -> 10000
[2026-01-08 00:30:49] train_sft.py:102 - [Rank 5] SFTTrainer initialized successfully
[2026-01-08 00:30:49] train_sft.py:701 - [Rank 5] Starting training: rollout_id 0 -> 10000

actor_val:   0%|          | 0/100 [00:00<?, ?it/s][2026-01-08 00:30:49] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:   1%|          | 1/100 [00:05<09:46,  5.93s/it][2026-01-08 00:30:55] train_sft.py:375 - SFT rollout sample: tokens_len=973, response_length=388

actor_val:   2%|▏         | 2/100 [00:06<04:52,  2.98s/it][2026-01-08 00:30:55] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:   3%|▎         | 3/100 [00:07<03:15,  2.01s/it][2026-01-08 00:30:56] train_sft.py:375 - SFT rollout sample: tokens_len=3806, response_length=2853

actor_val:   4%|▍         | 4/100 [00:08<02:26,  1.53s/it][2026-01-08 00:30:57] train_sft.py:375 - SFT rollout sample: tokens_len=3530, response_length=2945

actor_val:   5%|▌         | 5/100 [00:09<02:04,  1.31s/it][2026-01-08 00:30:58] train_sft.py:375 - SFT rollout sample: tokens_len=3721, response_length=3136

actor_val:   6%|▌         | 6/100 [00:10<01:41,  1.08s/it][2026-01-08 00:30:59] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:   7%|▋         | 7/100 [00:10<01:32,  1.00it/s][2026-01-08 00:31:00] train_sft.py:375 - SFT rollout sample: tokens_len=3505, response_length=2920

actor_val:   8%|▊         | 8/100 [00:11<01:18,  1.18it/s][2026-01-08 00:31:00] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:   9%|▉         | 9/100 [00:11<01:07,  1.34it/s][2026-01-08 00:31:01] train_sft.py:375 - SFT rollout sample: tokens_len=3465, response_length=2507

actor_val:  10%|█         | 10/100 [00:12<01:00,  1.48it/s][2026-01-08 00:31:01] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:  11%|█         | 11/100 [00:12<00:55,  1.59it/s][2026-01-08 00:31:02] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:  12%|█▏        | 12/100 [00:13<00:52,  1.67it/s][2026-01-08 00:31:02] train_sft.py:375 - SFT rollout sample: tokens_len=3530, response_length=2945

actor_val:  13%|█▎        | 13/100 [00:14<00:50,  1.74it/s][2026-01-08 00:31:03] train_sft.py:375 - SFT rollout sample: tokens_len=3505, response_length=2920

actor_val:  14%|█▍        | 14/100 [00:14<00:48,  1.77it/s][2026-01-08 00:31:03] train_sft.py:375 - SFT rollout sample: tokens_len=973, response_length=388

actor_val:  15%|█▌        | 15/100 [00:15<00:45,  1.86it/s][2026-01-08 00:31:04] train_sft.py:375 - SFT rollout sample: tokens_len=1932, response_length=1347

actor_val:  16%|█▌        | 16/100 [00:15<00:45,  1.86it/s][2026-01-08 00:31:04] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:  17%|█▋        | 17/100 [00:16<00:44,  1.85it/s][2026-01-08 00:31:05] train_sft.py:375 - SFT rollout sample: tokens_len=1932, response_length=1347

actor_val:  18%|█▊        | 18/100 [00:16<00:43,  1.90it/s][2026-01-08 00:31:05] train_sft.py:375 - SFT rollout sample: tokens_len=3721, response_length=3136

actor_val:  19%|█▉        | 19/100 [00:17<00:43,  1.87it/s][2026-01-08 00:31:06] train_sft.py:375 - SFT rollout sample: tokens_len=973, response_length=388

actor_val:  20%|██        | 20/100 [00:17<00:41,  1.93it/s][2026-01-08 00:31:06] train_sft.py:375 - SFT rollout sample: tokens_len=3282, response_length=2117

actor_val:  21%|██        | 21/100 [00:18<00:41,  1.90it/s][2026-01-08 00:31:07] train_sft.py:375 - SFT rollout sample: tokens_len=1932, response_length=1347

actor_val:  22%|██▏       | 22/100 [00:18<00:40,  1.93it/s][2026-01-08 00:31:07] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:  23%|██▎       | 23/100 [00:19<00:40,  1.91it/s][2026-01-08 00:31:08] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:  24%|██▍       | 24/100 [00:19<00:39,  1.90it/s][2026-01-08 00:31:08] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:  25%|██▌       | 25/100 [00:20<00:39,  1.90it/s][2026-01-08 00:31:09] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:  26%|██▌       | 26/100 [00:20<00:38,  1.92it/s][2026-01-08 00:31:09] train_sft.py:375 - SFT rollout sample: tokens_len=2743, response_length=2158

actor_val:  27%|██▋       | 27/100 [00:21<00:38,  1.92it/s][2026-01-08 00:31:10] train_sft.py:375 - SFT rollout sample: tokens_len=2106, response_length=1521

actor_val:  28%|██▊       | 28/100 [00:21<00:37,  1.92it/s][2026-01-08 00:31:10] train_sft.py:375 - SFT rollout sample: tokens_len=1932, response_length=1347

actor_val:  29%|██▉       | 29/100 [00:22<00:36,  1.93it/s][2026-01-08 00:31:11] train_sft.py:375 - SFT rollout sample: tokens_len=3721, response_length=3136

actor_val:  30%|███       | 30/100 [00:22<00:36,  1.91it/s][2026-01-08 00:31:12] train_sft.py:375 - SFT rollout sample: tokens_len=2743, response_length=2158

actor_val:  31%|███       | 31/100 [00:23<00:35,  1.92it/s][2026-01-08 00:31:12] train_sft.py:375 - SFT rollout sample: tokens_len=3505, response_length=2920

actor_val:  32%|███▏      | 32/100 [00:23<00:35,  1.92it/s][2026-01-08 00:31:13] train_sft.py:375 - SFT rollout sample: tokens_len=3505, response_length=2920

actor_val:  33%|███▎      | 33/100 [00:24<00:35,  1.91it/s][2026-01-08 00:31:13] train_sft.py:375 - SFT rollout sample: tokens_len=3806, response_length=2853

actor_val:  34%|███▍      | 34/100 [00:24<00:34,  1.91it/s][2026-01-08 00:31:14] train_sft.py:375 - SFT rollout sample: tokens_len=2106, response_length=1521

actor_val:  35%|███▌      | 35/100 [00:25<00:33,  1.92it/s][2026-01-08 00:31:14] train_sft.py:375 - SFT rollout sample: tokens_len=3282, response_length=2117

actor_val:  36%|███▌      | 36/100 [00:26<00:33,  1.91it/s][2026-01-08 00:31:15] train_sft.py:375 - SFT rollout sample: tokens_len=2743, response_length=2158

actor_val:  37%|███▋      | 37/100 [00:26<00:32,  1.91it/s][2026-01-08 00:31:15] train_sft.py:375 - SFT rollout sample: tokens_len=3505, response_length=2920

actor_val:  38%|███▊      | 38/100 [00:27<00:32,  1.90it/s][2026-01-08 00:31:16] train_sft.py:375 - SFT rollout sample: tokens_len=3505, response_length=2920

actor_val:  39%|███▉      | 39/100 [00:27<00:31,  1.91it/s][2026-01-08 00:31:16] train_sft.py:375 - SFT rollout sample: tokens_len=3721, response_length=3136

actor_val:  40%|████      | 40/100 [00:28<00:31,  1.92it/s][2026-01-08 00:31:17] train_sft.py:375 - SFT rollout sample: tokens_len=3530, response_length=2945

actor_val:  41%|████      | 41/100 [00:28<00:30,  1.92it/s][2026-01-08 00:31:17] train_sft.py:375 - SFT rollout sample: tokens_len=3806, response_length=2853

actor_val:  42%|████▏     | 42/100 [00:29<00:30,  1.91it/s][2026-01-08 00:31:18] train_sft.py:375 - SFT rollout sample: tokens_len=3806, response_length=2853

actor_val:  43%|████▎     | 43/100 [00:29<00:29,  1.91it/s][2026-01-08 00:31:18] train_sft.py:375 - SFT rollout sample: tokens_len=3282, response_length=2117

actor_val:  44%|████▍     | 44/100 [00:30<00:29,  1.93it/s][2026-01-08 00:31:19] train_sft.py:375 - SFT rollout sample: tokens_len=1932, response_length=1347

actor_val:  45%|████▌     | 45/100 [00:30<00:28,  1.94it/s][2026-01-08 00:31:19] train_sft.py:375 - SFT rollout sample: tokens_len=3721, response_length=3136

actor_val:  46%|████▌     | 46/100 [00:31<00:28,  1.92it/s][2026-01-08 00:31:20] train_sft.py:375 - SFT rollout sample: tokens_len=3806, response_length=2853

actor_val:  47%|████▋     | 47/100 [00:31<00:27,  1.91it/s][2026-01-08 00:31:20] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:  48%|████▊     | 48/100 [00:32<00:27,  1.92it/s][2026-01-08 00:31:21] train_sft.py:375 - SFT rollout sample: tokens_len=3465, response_length=2507

actor_val:  49%|████▉     | 49/100 [00:32<00:26,  1.92it/s][2026-01-08 00:31:21] train_sft.py:375 - SFT rollout sample: tokens_len=3806, response_length=2853

actor_val:  50%|█████     | 50/100 [00:33<00:26,  1.90it/s][2026-01-08 00:31:22] train_sft.py:375 - SFT rollout sample: tokens_len=3505, response_length=2920

actor_val:  51%|█████     | 51/100 [00:33<00:25,  1.90it/s][2026-01-08 00:31:23] train_sft.py:375 - SFT rollout sample: tokens_len=3282, response_length=2117

actor_val:  52%|█████▏    | 52/100 [00:34<00:25,  1.91it/s][2026-01-08 00:31:23] train_sft.py:375 - SFT rollout sample: tokens_len=3505, response_length=2920

actor_val:  53%|█████▎    | 53/100 [00:34<00:24,  1.90it/s][2026-01-08 00:31:24] train_sft.py:375 - SFT rollout sample: tokens_len=1932, response_length=1347

actor_val:  54%|█████▍    | 54/100 [00:35<00:23,  1.93it/s][2026-01-08 00:31:24] train_sft.py:375 - SFT rollout sample: tokens_len=2743, response_length=2158

actor_val:  55%|█████▌    | 55/100 [00:35<00:23,  1.91it/s][2026-01-08 00:31:25] train_sft.py:375 - SFT rollout sample: tokens_len=3282, response_length=2117

actor_val:  56%|█████▌    | 56/100 [00:36<00:23,  1.91it/s][2026-01-08 00:31:25] train_sft.py:375 - SFT rollout sample: tokens_len=3465, response_length=2507

actor_val:  57%|█████▋    | 57/100 [00:37<00:22,  1.91it/s][2026-01-08 00:31:26] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:  58%|█████▊    | 58/100 [00:37<00:22,  1.90it/s][2026-01-08 00:31:26] train_sft.py:375 - SFT rollout sample: tokens_len=3465, response_length=2507

actor_val:  59%|█████▉    | 59/100 [00:38<00:21,  1.93it/s][2026-01-08 00:31:27] train_sft.py:375 - SFT rollout sample: tokens_len=3530, response_length=2945

actor_val:  60%|██████    | 60/100 [00:38<00:20,  1.92it/s][2026-01-08 00:31:27] train_sft.py:375 - SFT rollout sample: tokens_len=3282, response_length=2117

actor_val:  61%|██████    | 61/100 [00:39<00:20,  1.91it/s][2026-01-08 00:31:28] train_sft.py:375 - SFT rollout sample: tokens_len=973, response_length=388

actor_val:  62%|██████▏   | 62/100 [00:39<00:19,  1.96it/s][2026-01-08 00:31:28] train_sft.py:375 - SFT rollout sample: tokens_len=3282, response_length=2117

actor_val:  63%|██████▎   | 63/100 [00:40<00:19,  1.92it/s][2026-01-08 00:31:29] train_sft.py:375 - SFT rollout sample: tokens_len=2743, response_length=2158

actor_val:  64%|██████▍   | 64/100 [00:40<00:18,  1.92it/s][2026-01-08 00:31:29] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:  65%|██████▌   | 65/100 [00:41<00:18,  1.92it/s][2026-01-08 00:31:30] train_sft.py:375 - SFT rollout sample: tokens_len=1932, response_length=1347

actor_val:  66%|██████▌   | 66/100 [00:41<00:17,  1.94it/s][2026-01-08 00:31:30] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:  67%|██████▋   | 67/100 [00:42<00:17,  1.92it/s][2026-01-08 00:31:31] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:  68%|██████▊   | 68/100 [00:42<00:16,  1.91it/s][2026-01-08 00:31:31] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:  69%|██████▉   | 69/100 [00:43<00:16,  1.91it/s][2026-01-08 00:31:32] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:  70%|███████   | 70/100 [00:43<00:15,  1.93it/s][2026-01-08 00:31:32] train_sft.py:375 - SFT rollout sample: tokens_len=1932, response_length=1347

actor_val:  71%|███████   | 71/100 [00:44<00:14,  1.94it/s][2026-01-08 00:31:33] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:  72%|███████▏  | 72/100 [00:44<00:14,  1.92it/s][2026-01-08 00:31:33] train_sft.py:375 - SFT rollout sample: tokens_len=3721, response_length=3136

actor_val:  73%|███████▎  | 73/100 [00:45<00:14,  1.91it/s][2026-01-08 00:31:34] train_sft.py:375 - SFT rollout sample: tokens_len=973, response_length=388

actor_val:  74%|███████▍  | 74/100 [00:45<00:13,  1.96it/s][2026-01-08 00:31:34] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:  75%|███████▌  | 75/100 [00:46<00:13,  1.90it/s][2026-01-08 00:31:35] train_sft.py:375 - SFT rollout sample: tokens_len=3530, response_length=2945

actor_val:  76%|███████▌  | 76/100 [00:46<00:12,  1.90it/s][2026-01-08 00:31:36] train_sft.py:375 - SFT rollout sample: tokens_len=3465, response_length=2507

actor_val:  77%|███████▋  | 77/100 [00:47<00:12,  1.91it/s][2026-01-08 00:31:36] train_sft.py:375 - SFT rollout sample: tokens_len=3465, response_length=2507

actor_val:  78%|███████▊  | 78/100 [00:47<00:11,  1.91it/s][2026-01-08 00:31:37] train_sft.py:375 - SFT rollout sample: tokens_len=3806, response_length=2853

actor_val:  79%|███████▉  | 79/100 [00:48<00:11,  1.90it/s][2026-01-08 00:31:37] train_sft.py:375 - SFT rollout sample: tokens_len=973, response_length=388

actor_val:  80%|████████  | 80/100 [00:48<00:10,  1.95it/s][2026-01-08 00:31:38] train_sft.py:375 - SFT rollout sample: tokens_len=2106, response_length=1521

actor_val:  81%|████████  | 81/100 [00:49<00:09,  1.93it/s][2026-01-08 00:31:38] train_sft.py:375 - SFT rollout sample: tokens_len=3465, response_length=2507

actor_val:  82%|████████▏ | 82/100 [00:50<00:09,  1.91it/s][2026-01-08 00:31:39] train_sft.py:375 - SFT rollout sample: tokens_len=3505, response_length=2920

actor_val:  83%|████████▎ | 83/100 [00:50<00:08,  1.91it/s][2026-01-08 00:31:39] train_sft.py:375 - SFT rollout sample: tokens_len=3282, response_length=2117

actor_val:  84%|████████▍ | 84/100 [00:51<00:08,  1.91it/s][2026-01-08 00:31:40] train_sft.py:375 - SFT rollout sample: tokens_len=973, response_length=388

actor_val:  85%|████████▌ | 85/100 [00:51<00:07,  1.95it/s][2026-01-08 00:31:40] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:  86%|████████▌ | 86/100 [00:52<00:07,  1.91it/s][2026-01-08 00:31:41] train_sft.py:375 - SFT rollout sample: tokens_len=3721, response_length=3136

actor_val:  87%|████████▋ | 87/100 [00:52<00:06,  1.90it/s][2026-01-08 00:31:41] train_sft.py:375 - SFT rollout sample: tokens_len=3530, response_length=2945

actor_val:  88%|████████▊ | 88/100 [00:53<00:06,  1.89it/s][2026-01-08 00:31:42] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:  89%|████████▉ | 89/100 [00:53<00:05,  1.92it/s][2026-01-08 00:31:42] train_sft.py:375 - SFT rollout sample: tokens_len=973, response_length=388

actor_val:  90%|█████████ | 90/100 [00:54<00:05,  1.94it/s][2026-01-08 00:31:43] train_sft.py:375 - SFT rollout sample: tokens_len=3721, response_length=3136

actor_val:  91%|█████████ | 91/100 [00:54<00:04,  1.89it/s][2026-01-08 00:31:43] train_sft.py:375 - SFT rollout sample: tokens_len=3806, response_length=2853

actor_val:  92%|█████████▏| 92/100 [00:55<00:04,  1.90it/s][2026-01-08 00:31:44] train_sft.py:375 - SFT rollout sample: tokens_len=2106, response_length=1521

actor_val:  93%|█████████▎| 93/100 [00:55<00:03,  1.92it/s][2026-01-08 00:31:44] train_sft.py:375 - SFT rollout sample: tokens_len=973, response_length=388

actor_val:  94%|█████████▍| 94/100 [00:56<00:03,  1.94it/s][2026-01-08 00:31:45] train_sft.py:375 - SFT rollout sample: tokens_len=2106, response_length=1521

actor_val:  95%|█████████▌| 95/100 [00:56<00:02,  1.91it/s][2026-01-08 00:31:45] train_sft.py:375 - SFT rollout sample: tokens_len=3465, response_length=2507

actor_val:  96%|█████████▌| 96/100 [00:57<00:02,  1.88it/s][2026-01-08 00:31:46] train_sft.py:375 - SFT rollout sample: tokens_len=973, response_length=388

actor_val:  97%|█████████▋| 97/100 [00:57<00:01,  1.93it/s][2026-01-08 00:31:46] train_sft.py:375 - SFT rollout sample: tokens_len=2743, response_length=2158

actor_val:  98%|█████████▊| 98/100 [00:58<00:01,  1.90it/s][2026-01-08 00:31:47] train_sft.py:375 - SFT rollout sample: tokens_len=3530, response_length=2945

actor_val:  99%|█████████▉| 99/100 [00:58<00:00,  1.90it/s][2026-01-08 00:31:48] train_sft.py:375 - SFT rollout sample: tokens_len=3505, response_length=2920

actor_val: 100%|██████████| 100/100 [00:59<00:00,  1.89it/s]
actor_val: 100%|██████████| 100/100 [00:59<00:00,  1.68it/s]
[2026-01-08 00:31:48] train_sft.py:657 - step 0: {'val/loss': 3.3324711990356444}
[2026-01-08 00:31:48] train_sft.py:375 - SFT rollout sample: tokens_len=4043, response_length=3458
[2026-01-08 00:31:48] timer.py:24 - Timer actor_train start

actor_train:   0%|          | 0/1 [00:00<?, ?it/s]
actor_train:   0%|          | 0/1 [00:07<?, ?it/s]
[2026-01-08 00:31:55] timer.py:32 - Timer actor_train end (elapsed: 7.3s)
Traceback (most recent call last):
  File "/p/project1/envcomp/mahajan1/miles/train_sft.py", line 756, in <module>
    main()
  File "/p/project1/envcomp/mahajan1/miles/train_sft.py", line 752, in main
    trainer.train()
  File "/p/project1/envcomp/mahajan1/miles/train_sft.py", line 714, in train
    self.train_one_rollout(rollout_id)
  File "/p/project1/envcomp/mahajan1/miles/train_sft.py", line 622, in train_one_rollout
    self._train_step(packed_batch, reported_accum, mbs_id, grad_accum)
  File "/p/project1/envcomp/mahajan1/miles/train_sft.py", line 559, in _train_step
    loss.backward()
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_tensor.py", line 647, in backward
    torch.autograd.backward(
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/autograd/__init__.py", line 354, in backward
    _engine_run_backward(
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/autograd/graph.py", line 829, in _engine_run_backward
    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/autograd/function.py", line 311, in apply
    return user_fn(self, *args)
           ^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2259, in backward
    return impl_fn()
           ^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2245, in impl_fn
    out = CompiledFunction._backward_impl(ctx, all_args)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2376, in _backward_impl
    out = call_func_at_runtime_with_args(
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args
    out = normalize_as_list(f(args))
                            ^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 929, in _fn
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_inductor/output_code.py", line 584, in __call__
    return self.current_callable(inputs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_inductor/utils.py", line 2716, in run
    out = model(new_inputs)
          ^^^^^^^^^^^^^^^^^
  File "/tmp/torchinductor_mahajan1/mz/cmz6z6omwxghwptvrh3pepwvm5yclw24uq7kvx5kpuyb733ecbb6.py", line 198, in call
    buf3 = empty_strided_cuda((s88, s3), (s3, 1), torch.float32)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.29 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1.21 GiB is free. Including non-PyTorch memory, this process has 38.27 GiB memory in use. Of the allocated memory 37.21 GiB is allocated by PyTorch, and 266.90 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
[rank0]: Traceback (most recent call last):
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/train_sft.py", line 756, in <module>
[rank0]:     main()
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/train_sft.py", line 752, in main
[rank0]:     trainer.train()
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/train_sft.py", line 714, in train
[rank0]:     self.train_one_rollout(rollout_id)
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/train_sft.py", line 622, in train_one_rollout
[rank0]:     self._train_step(packed_batch, reported_accum, mbs_id, grad_accum)
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/train_sft.py", line 559, in _train_step
[rank0]:     loss.backward()
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_tensor.py", line 647, in backward
[rank0]:     torch.autograd.backward(
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/autograd/__init__.py", line 354, in backward
[rank0]:     _engine_run_backward(
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/autograd/graph.py", line 829, in _engine_run_backward
[rank0]:     return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/autograd/function.py", line 311, in apply
[rank0]:     return user_fn(self, *args)
[rank0]:            ^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2259, in backward
[rank0]:     return impl_fn()
[rank0]:            ^^^^^^^^^
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2245, in impl_fn
[rank0]:     out = CompiledFunction._backward_impl(ctx, all_args)
[rank0]:           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2376, in _backward_impl
[rank0]:     out = call_func_at_runtime_with_args(
[rank0]:           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args
[rank0]:     out = normalize_as_list(f(args))
[rank0]:                             ^^^^^^^
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 929, in _fn
[rank0]:     return fn(*args, **kwargs)
[rank0]:            ^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_inductor/output_code.py", line 584, in __call__
[rank0]:     return self.current_callable(inputs)
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/_inductor/utils.py", line 2716, in run
[rank0]:     out = model(new_inputs)
[rank0]:           ^^^^^^^^^^^^^^^^^
[rank0]:   File "/tmp/torchinductor_mahajan1/mz/cmz6z6omwxghwptvrh3pepwvm5yclw24uq7kvx5kpuyb733ecbb6.py", line 198, in call
[rank0]:     buf3 = empty_strided_cuda((s88, s3), (s3, 1), torch.float32)
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.29 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1.21 GiB is free. Including non-PyTorch memory, this process has 38.27 GiB memory in use. Of the allocated memory 37.21 GiB is allocated by PyTorch, and 266.90 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
wandb: 
wandb: You can sync this run to the cloud by running:
wandb: wandb sync /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS/wandb/offline-run-20260108_002832-37m2paam
wandb: Find logs at: ../../../../../../../scratch/envcomp/logs/BROKEN_RESOLUTIONS/wandb/offline-run-20260108_002832-37m2paam/logs
W0108 00:32:01.009000 622874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 622922 closing signal SIGTERM
W0108 00:32:01.015000 622874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 622923 closing signal SIGTERM
W0108 00:32:01.016000 622874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 622924 closing signal SIGTERM
E0108 00:32:04.076000 622874 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 0 (pid: 622921) of binary: /p/project1/envcomp/mahajan1/miles/.venv/bin/python3
Traceback (most recent call last):
  File "/p/project1/envcomp/mahajan1/miles/.venv/bin/torchrun", line 10, in <module>
    sys.exit(main())
             ^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
    return f(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/run.py", line 901, in main
    run(args)
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/run.py", line 892, in run
    elastic_launch(
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 143, in __call__
    return launch_agent(self._config, self._entrypoint, list(args))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent
    raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
============================================================
train_sft.py FAILED
------------------------------------------------------------
Failures:
  <NO_OTHER_FAILURES>
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
  time      : 2026-01-08_00:32:01
  host      : jwb0757.juwels
  rank      : 0 (local_rank: 0)
  exitcode  : 1 (pid: 622921)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
============================================================
W0108 00:32:04.395000 254659 torch/distributed/elastic/agent/server/api.py:723] Received 15 death signal, shutting down workers
W0108 00:32:04.402000 254659 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 254696 closing signal SIGTERM
W0108 00:32:04.404000 254659 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 254697 closing signal SIGTERM
W0108 00:32:04.405000 254659 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 254698 closing signal SIGTERM
W0108 00:32:04.406000 254659 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 254699 closing signal SIGTERM
Traceback (most recent call last):
  File "/p/project1/envcomp/mahajan1/miles/.venv/bin/torchrun", line 10, in <module>
    sys.exit(main())
             ^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
    return f(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/run.py", line 901, in main
    run(args)
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/run.py", line 892, in run
    elastic_launch(
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 143, in __call__
    return launch_agent(self._config, self._entrypoint, list(args))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 268, in launch_agent
    result = agent.run()
             ^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
    result = f(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/agent/server/api.py", line 715, in run
    result = self._invoke_run(role)
             ^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/agent/server/api.py", line 879, in _invoke_run
    time.sleep(monitor_interval)
  File "/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
torch.distributed.elastic.multiprocessing.api.SignalException: Process 254659 got signal: 15
srun: error: jwb0759: task 1: Terminated
srun: error: jwb0757: task 0: Exited with exit code 1
srun: Force Terminated StepId=13068223.0
srun execution finished.