[tab] /p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/qwen_4b_lora_bs_16_13068224.log (log)
#!/usr/bin/env bash
#
# SBATCH Template Script for SFT Training
# This script is invoked by the launcher with environment variables set
#
# Required environment variables (set by launcher):
#   - MODEL_SIZE: 1.7b, 4b, or 8b
#   - BATCH_SIZE: 8, 16, or 32
#   - USE_LORA: true or false
#

# #SBATCH --nodes=4  # Set dynamically by launcher
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --partition=booster
#SBATCH --gres=gpu:4
#SBATCH --mem=200GB
#SBATCH --account=envcomp
#SBATCH --nodelist=jwb[0700-0799]
#SBATCH --signal=USR1@60
#SBATCH --output=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log

trap 'echo "Job killed by signal"' USR1

# Print this script for debugging
cat $0

echo "============================================"
echo "SBATCH Environment:"
echo "  MODEL_SIZE: ${MODEL_SIZE}"
echo "  BATCH_SIZE: ${BATCH_SIZE}"
echo "  USE_LORA:   ${USE_LORA}"
echo "  CP_SIZE:    ${CONTEXT_PARALLEL_SIZE}"
echo "  SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "============================================"

# Load required modules
module load Stages/2025
module load CUDA/12
module load cuDNN/9.5.0.50-CUDA-12
module load NCCL/default-CUDA-12
module load Clang/18.1.8

# Activate virtual environment
source .venv/bin/activate

# Determine Master Address once to avoid multiple scontrol calls
export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
export MASTER_PORT=29500
echo "============================================"
echo "Master node is ${MASTER_ADDR} on port ${MASTER_PORT}"
echo "============================================"

# Export configuration for the run script
export MODEL_SIZE="${MODEL_SIZE}"
export BATCH_SIZE="${BATCH_SIZE}"
export USE_LORA="${USE_LORA}"
export CONTEXT_PARALLEL_SIZE="${CONTEXT_PARALLEL_SIZE}"

echo "Launching training script on all nodes via srun..."
# Run the training script on all nodes
srun sh slurm/jobs/mihir/juelich/crowd-pilot/2-wave-broken-resolutions/run_template.sh
echo "srun execution finished."
============================================
SBATCH Environment:
  MODEL_SIZE: 4b
  BATCH_SIZE: 16
  USE_LORA:   true
  CP_SIZE:    1
  SLURM_JOB_ID: 13068224
============================================

  This stage is deprecated. Please consider moving to a new stage (2026
or newer)


Due to MODULEPATH changes, the following have been reloaded:
  1) make/4.4.1

The following have been reloaded with a version change:
  1) GCCcore/.14.3.0 => GCCcore/.13.3.0     4) binutils/2.44 => binutils/.2.42
  2) Stages/2026 => Stages/2025             5) zlib/1.3.1 => zlib/.1.3.1
  3) StdEnv/2026 => StdEnv/2025

============================================
Master node is jwb0776 on port 29500
============================================
Launching training script on all nodes via srun...
Environment variables have been set
Environment variables have been set
Environment variables have been set
Environment variables have been set
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  16
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     125
  NUM_ROLL:    5000
  LR_WARMUP:   250
  RUN_NAME:    qwen4B_lora_bs_16_13068224
  RUN_ID:      13068224
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068224/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_16_13068224 --wandb-run-id 13068224 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  16
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     125
  NUM_ROLL:    5000
  LR_WARMUP:   250
  RUN_NAME:    qwen4B_lora_bs_16_13068224
  RUN_ID:      13068224
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068224/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_16_13068224 --wandb-run-id 13068224 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  16
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     125
  NUM_ROLL:    5000
  LR_WARMUP:   250
  RUN_NAME:    qwen4B_lora_bs_16_13068224
  RUN_ID:      13068224
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068224/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_16_13068224 --wandb-run-id 13068224 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  16
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     125
  NUM_ROLL:    5000
  LR_WARMUP:   250
  RUN_NAME:    qwen4B_lora_bs_16_13068224
  RUN_ID:      13068224
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068224/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_16_13068224 --wandb-run-id 13068224 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
W0107 22:25:53.015000 88279 torch/distributed/run.py:774] 
W0107 22:25:53.015000 88279 torch/distributed/run.py:774] *****************************************
W0107 22:25:53.015000 88279 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 22:25:53.015000 88279 torch/distributed/run.py:774] *****************************************
[W107 22:25:53.173977417 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
W0107 22:25:53.633000 2775794 torch/distributed/run.py:774] 
W0107 22:25:53.633000 2775794 torch/distributed/run.py:774] *****************************************
W0107 22:25:53.633000 2775794 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 22:25:53.633000 2775794 torch/distributed/run.py:774] *****************************************
[W107 22:25:53.835996058 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
W0107 22:25:54.092000 1571683 torch/distributed/run.py:774] 
W0107 22:25:54.092000 1571683 torch/distributed/run.py:774] *****************************************
W0107 22:25:54.092000 1571683 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 22:25:54.092000 1571683 torch/distributed/run.py:774] *****************************************
[W107 22:25:54.246229639 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
W0107 22:25:54.226000 2808683 torch/distributed/run.py:774] 
W0107 22:25:54.226000 2808683 torch/distributed/run.py:774] *****************************************
W0107 22:25:54.226000 2808683 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 22:25:54.226000 2808683 torch/distributed/run.py:774] *****************************************
[W107 22:25:54.436397572 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
[2026-01-07 22:26:27] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:27] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:27] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:27] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:27] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:27] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:27] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:27] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[W107 22:26:28.780801993 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:28.781587778 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:28.784025055 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:28.784393007 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:28.797875758 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:28.798315101 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:28.799445008 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:28.799800240 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[2026-01-07 22:26:30] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:30] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:30] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:30] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:30] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:30] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:30] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:30] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:30] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:30] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:30] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:30] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:30] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:30] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:30] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:30] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[W107 22:26:30.917010176 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:30.917154906 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:30.918057920 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:30.918059260 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:30.934279871 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:30.934490482 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:30.935059474 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:30.935193875 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:31.421467592 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:31.422497286 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:31.432099460 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:31.432299421 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:31.432908603 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:31.433061364 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:31.436247105 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:31.436912817 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[2026-01-07 22:26:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:33] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 22:26:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 22:26:33] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[W107 22:26:33.935294894 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:33.936341052 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:33.936735754 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:33.936780325 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:33.936817855 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0776.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 22:26:33.937378949 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:33.937450400 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 22:26:33.937506050 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[Gloo] Rank 0 is connected to 15 peer ranks. Expected number of connected peer ranks is : 15
[Gloo] Rank 1 is connected to 15 peer ranks. Expected number of connected peer ranks is : 15
[Gloo] Rank 2 is connected to 15 peer ranks. Expected number of connected peer ranks is : 15
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 0] Distributed initialized: world_size=16, local_rank=0
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 1] Distributed initialized: world_size=16, local_rank=1
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 2] Distributed initialized: world_size=16, local_rank=2
[Gloo] Rank 4 is connected to 15 peer ranks. Expected number of connected peer ranks is : 15
[Gloo] Rank 5 is connected to 15 peer ranks. Expected number of connected peer ranks is : 15
[Gloo] Rank 3 is connected to 15 peer ranks. Expected number of connected peer ranks is : 15
[Gloo] Rank [Gloo] Rank 7 is connected to 156 peer ranks. Expected number of connected peer ranks is :  is connected to 1515
 peer ranks. Expected number of connected peer ranks is : 15
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 4] Distributed initialized: world_size=16, local_rank=0
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 5] Distributed initialized: world_size=16, local_rank=1
[Gloo] Rank 8 is connected to 15 peer ranks. Expected number of connected peer ranks is : 15
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 3] Distributed initialized: world_size=16, local_rank=3
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 7] Distributed initialized: world_size=16, local_rank=3
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 6] Distributed initialized: world_size=16, local_rank=2
[Gloo] Rank 13 is connected to [Gloo] Rank 15 peer ranks. 12Expected number of connected peer ranks is :  is connected to 1515 peer ranks. 
Expected number of connected peer ranks is : 15
[Gloo] Rank 9 is connected to 15 peer ranks. Expected number of connected peer ranks is : 15
[Gloo] Rank 10 is connected to 15 peer ranks. Expected number of connected peer ranks is : 15
[Gloo] Rank 11 is connected to 15 peer ranks. Expected number of connected peer ranks is : 15
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 9] Distributed initialized: world_size=16, local_rank=1
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 8] Distributed initialized: world_size=16, local_rank=0
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 10] Distributed initialized: world_size=16, local_rank=2
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 13] Distributed initialized: world_size=16, local_rank=1
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 12] Distributed initialized: world_size=16, local_rank=0
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 11] Distributed initialized: world_size=16, local_rank=3
[Gloo] Rank [Gloo] Rank 1415 is connected to  is connected to 1515 peer ranks.  peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 1515

[2026-01-07 22:26:33] train_sft.py:120 - [Rank 14] Distributed initialized: world_size=16, local_rank=2
[2026-01-07 22:26:33] train_sft.py:120 - [Rank 15] Distributed initialized: world_size=16, local_rank=3
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 0] Device mesh: dp_size=16, cp_size=1, dp_rank=0, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 4] Device mesh: dp_size=16, cp_size=1, dp_rank=4, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 7] Device mesh: dp_size=16, cp_size=1, dp_rank=7, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 5] Device mesh: dp_size=16, cp_size=1, dp_rank=5, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 6] Device mesh: dp_size=16, cp_size=1, dp_rank=6, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 1] Device mesh: dp_size=16, cp_size=1, dp_rank=1, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 11] Device mesh: dp_size=16, cp_size=1, dp_rank=11, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 8] Device mesh: dp_size=16, cp_size=1, dp_rank=8, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 9] Device mesh: dp_size=16, cp_size=1, dp_rank=9, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 10] Device mesh: dp_size=16, cp_size=1, dp_rank=10, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 12] Device mesh: dp_size=16, cp_size=1, dp_rank=12, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 13] Device mesh: dp_size=16, cp_size=1, dp_rank=13, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 15] Device mesh: dp_size=16, cp_size=1, dp_rank=15, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 14] Device mesh: dp_size=16, cp_size=1, dp_rank=14, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 2] Device mesh: dp_size=16, cp_size=1, dp_rank=2, cp_rank=0
[2026-01-07 22:26:33] train_sft.py:146 - [Rank 3] Device mesh: dp_size=16, cp_size=1, dp_rank=3, cp_rank=0
[2026-01-07 22:26:33] wandb_utils.py:72 - W&B logs will be stored in: /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
wandb: Tracking run with wandb version 0.23.1
wandb: W&B syncing is set to `offline` in this directory. Run `wandb online` or set WANDB_MODE=online to enable cloud syncing.
wandb: Run data is saved locally in /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS/wandb/offline-run-20260107_222633-slf57s4g
wandb: Detected [openai] in use.
wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
[2026-01-07 22:28:13] train_sft.py:218 - [Rank 15] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 22:28:13] train_sft.py:218 - [Rank 6] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 22:28:13] train_sft.py:218 - [Rank 9] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 22:28:14] train_sft.py:218 - [Rank 11] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 22:28:14] train_sft.py:218 - [Rank 8] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 22:28:14] train_sft.py:218 - [Rank 10] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 22:28:14] train_sft.py:218 - [Rank 13] Data source initialized: dataset_size=97, num_rollout=5000
[2026-01-07 22:28:14] train_sft.py:218 - [Rank 7] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 22:28:15] train_sft.py:218 - [Rank 5] Data source initialized: dataset_size=97, num_rollout=5000
[2026-01-07 22:28:15] train_sft.py:218 - [Rank 4] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 22:28:15] train_sft.py:218 - [Rank 2] Data source initialized: dataset_size=97, num_rollout=5000
[2026-01-07 22:28:15] train_sft.py:218 - [Rank 0] Data source initialized: dataset_size=97, num_rollout=5000
[2026-01-07 22:28:15] train_sft.py:218 - [Rank 1] Data source initialized: dataset_size=97, num_rollout=5000
[2026-01-07 22:28:15] train_sft.py:218 - [Rank 3] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 22:28:15] train_sft.py:218 - [Rank 14] Data source initialized: dataset_size=97, num_rollout=5000
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 22:28:15] train_sft.py:218 - [Rank 12] Data source initialized: dataset_size=97, num_rollout=5000

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:08<00:17,  8.68s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:20, 10.24s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:08<00:17,  8.76s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:08<00:17,  8.75s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:09<00:18,  9.15s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:09<00:19,  9.89s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:09<00:18,  9.13s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.74s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.71s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.77s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.86s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:12<00:25, 12.64s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.73s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.74s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.75s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.80s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:22<00:11, 11.92s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:22<00:11, 11.90s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:22<00:11, 11.91s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:24<00:12, 12.57s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  6.76s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  7.84s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  6.74s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  6.73s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  7.81s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  7.81s/it]


Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  7.08s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  8.33s/it]
[2026-01-07 22:28:38] train_sft.py:299 - [Rank 5] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:38] train_sft.py:299 - [Rank 4] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:38] train_sft.py:299 - [Rank 7] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:38] train_sft.py:299 - [Rank 6] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:38] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:38] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:38] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:38] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32

Loading checkpoint shards:  67%|██████▋   | 2/3 [00:23<00:12, 12.15s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:13, 13.22s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:23<00:12, 12.19s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:24<00:12, 12.49s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.36s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.79s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  6.80s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  6.79s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  7.94s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  7.95s/it]


Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  6.96s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  8.19s/it]
[2026-01-07 22:28:39] train_sft.py:299 - [Rank 12] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:39] train_sft.py:299 - [Rank 15] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:39] train_sft.py:299 - [Rank 14] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:39] train_sft.py:299 - [Rank 13] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:39] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:39] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:39] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:39] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32

Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.88s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.94s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.23s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.93s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.19s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.62s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.38s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.19s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.90s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.61s/it]


Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.22s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.65s/it]
[2026-01-07 22:28:40] train_sft.py:299 - [Rank 9] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:40] train_sft.py:299 - [Rank 11] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:40] train_sft.py:299 - [Rank 10] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:40] train_sft.py:299 - [Rank 8] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32

Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.78s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.79s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.78s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:27<00:13, 13.82s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  7.67s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  9.12s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  7.66s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  7.67s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  9.12s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  9.12s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  7.66s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  9.12s/it]
[2026-01-07 22:28:43] train_sft.py:299 - [Rank 3] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:43] train_sft.py:299 - [Rank 0] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:43] train_sft.py:299 - [Rank 1] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:43] train_sft.py:299 - [Rank 2] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 22:28:43] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:43] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:43] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:43] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 9] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 8] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 9] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 8] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 14] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 13] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 14] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 13] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 10] Model initialized with FSDP
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 10] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 11] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 12] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 11] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 12] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 15] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 15] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 7] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 2] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 7] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 2] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 4] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 6] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 4] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 6] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 5] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 5] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 3] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:321 - [Rank 1] Model initialized with FSDP
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 3] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] train_sft.py:330 - [Rank 1] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 22:28:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:50] train_sft.py:321 - [Rank 0] Model initialized with FSDP
[2026-01-07 22:28:50] train_sft.py:330 - [Rank 0] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 22:28:50] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 22:28:50] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 3] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 0] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 3] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 2] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 1] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 1] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 2] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 0] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 11] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 10] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 8] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 13] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 12] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 11] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 10] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 8] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 9] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 15] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 13] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 12] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 15] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 6] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 5] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 4] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 7] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 9] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 7] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 4] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 6] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 5] Starting training: rollout_id 0 -> 5000
[2026-01-07 22:28:50] train_sft.py:102 - [Rank 14] SFTTrainer initialized successfully
[2026-01-07 22:28:50] train_sft.py:701 - [Rank 14] Starting training: rollout_id 0 -> 5000

actor_val:   0%|          | 0/100 [00:00<?, ?it/s][2026-01-07 22:28:51] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960

actor_val:   1%|          | 1/100 [00:04<06:58,  4.23s/it][2026-01-07 22:28:55] train_sft.py:375 - SFT rollout sample: tokens_len=3013, response_length=2428

actor_val:   2%|▏         | 2/100 [00:05<03:57,  2.42s/it][2026-01-07 22:28:56] train_sft.py:375 - SFT rollout sample: tokens_len=3530, response_length=2945

actor_val:   3%|▎         | 3/100 [00:06<02:48,  1.73s/it][2026-01-07 22:28:57] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960
[2026-01-07 22:28:57] train_sft.py:637 - [Rank 15] No batches to validate on rollout 0, validation step 3
[2026-01-07 22:28:57] train_sft.py:637 - [Rank 14] No batches to validate on rollout 0, validation step 3
[2026-01-07 22:28:57] train_sft.py:637 - [Rank 12] No batches to validate on rollout 0, validation step 3
[2026-01-07 22:28:57] train_sft.py:637 - [Rank 13] No batches to validate on rollout 0, validation step 3

actor_val:   4%|▍         | 4/100 [00:07<02:18,  1.45s/it][2026-01-07 22:28:58] train_sft.py:375 - SFT rollout sample: tokens_len=3282, response_length=2117