[tab] /p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/qwen_4b_lora_bs_32_13068226.log (log)
#!/usr/bin/env bash
#
# SBATCH Template Script for SFT Training
# This script is invoked by the launcher with environment variables set
#
# Required environment variables (set by launcher):
#   - MODEL_SIZE: 1.7b, 4b, or 8b
#   - BATCH_SIZE: 8, 16, or 32
#   - USE_LORA: true or false
#

# #SBATCH --nodes=4  # Set dynamically by launcher
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --partition=booster
#SBATCH --gres=gpu:4
#SBATCH --mem=200GB
#SBATCH --account=envcomp
#SBATCH --nodelist=jwb[0700-0799]
#SBATCH --signal=USR1@60
#SBATCH --output=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log

trap 'echo "Job killed by signal"' USR1

# Print this script for debugging
cat $0

echo "============================================"
echo "SBATCH Environment:"
echo "  MODEL_SIZE: ${MODEL_SIZE}"
echo "  BATCH_SIZE: ${BATCH_SIZE}"
echo "  USE_LORA:   ${USE_LORA}"
echo "  CP_SIZE:    ${CONTEXT_PARALLEL_SIZE}"
echo "  SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "============================================"

# Load required modules
module load Stages/2025
module load CUDA/12
module load cuDNN/9.5.0.50-CUDA-12
module load NCCL/default-CUDA-12
module load Clang/18.1.8

# Activate virtual environment
source .venv/bin/activate

# Determine Master Address once to avoid multiple scontrol calls
export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
export MASTER_PORT=29500
echo "============================================"
echo "Master node is ${MASTER_ADDR} on port ${MASTER_PORT}"
echo "============================================"

# Export configuration for the run script
export MODEL_SIZE="${MODEL_SIZE}"
export BATCH_SIZE="${BATCH_SIZE}"
export USE_LORA="${USE_LORA}"
export CONTEXT_PARALLEL_SIZE="${CONTEXT_PARALLEL_SIZE}"

echo "Launching training script on all nodes via srun..."
# Run the training script on all nodes
srun sh slurm/jobs/mihir/juelich/crowd-pilot/2-wave-broken-resolutions/run_template.sh
echo "srun execution finished."
============================================
SBATCH Environment:
  MODEL_SIZE: 4b
  BATCH_SIZE: 32
  USE_LORA:   true
  CP_SIZE:    1
  SLURM_JOB_ID: 13068226
============================================

  This stage is deprecated. Please consider moving to a new stage (2026
or newer)


Due to MODULEPATH changes, the following have been reloaded:
  1) make/4.4.1

The following have been reloaded with a version change:
  1) GCCcore/.14.3.0 => GCCcore/.13.3.0     4) binutils/2.44 => binutils/.2.42
  2) Stages/2026 => Stages/2025             5) zlib/1.3.1 => zlib/.1.3.1
  3) StdEnv/2026 => StdEnv/2025

============================================
Master node is jwb0701 on port 29500
============================================
Launching training script on all nodes via srun...
Environment variables have been set
Environment variables have been set
Environment variables have been set
Environment variables have been set
Environment variables have been set
Environment variables have been set
Environment variables have been set
Environment variables have been set
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  32
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     62
  NUM_ROLL:    2500
  LR_WARMUP:   125
  RUN_NAME:    qwen4B_lora_bs_32_13068226
  RUN_ID:      13068226
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068226/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_32_13068226 --wandb-run-id 13068226 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  32
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     62
  NUM_ROLL:    2500
  LR_WARMUP:   125
  RUN_NAME:    qwen4B_lora_bs_32_13068226
  RUN_ID:      13068226
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068226/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_32_13068226 --wandb-run-id 13068226 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
HAS_NVLINK: 0 (detected 0 NVLink references)
HAS_NVLINK: 0 (detected 0 NVLink references)
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  32
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     62
  NUM_ROLL:    2500
  LR_WARMUP:   125
  RUN_NAME:    qwen4B_lora_bs_32_13068226
  RUN_ID:      13068226
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068226/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_32_13068226 --wandb-run-id 13068226 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  32
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     62
  NUM_ROLL:    2500
  LR_WARMUP:   125
  RUN_NAME:    qwen4B_lora_bs_32_13068226
  RUN_ID:      13068226
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068226/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_32_13068226 --wandb-run-id 13068226 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  32
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     62
  NUM_ROLL:    2500
  LR_WARMUP:   125
  RUN_NAME:    qwen4B_lora_bs_32_13068226
  RUN_ID:      13068226
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068226/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_32_13068226 --wandb-run-id 13068226 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  32
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     62
  NUM_ROLL:    2500
  LR_WARMUP:   125
  RUN_NAME:    qwen4B_lora_bs_32_13068226
  RUN_ID:      13068226
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068226/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_32_13068226 --wandb-run-id 13068226 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  32
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     62
  NUM_ROLL:    2500
  LR_WARMUP:   125
  RUN_NAME:    qwen4B_lora_bs_32_13068226
  RUN_ID:      13068226
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068226/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_32_13068226 --wandb-run-id 13068226 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
HAS_NVLINK: 0 (detected 0 NVLink references)
Cluster config successful
Model has been set: Qwen3-4B
============================================
Run Configuration:
  MODEL_SIZE:  4b
  MODEL_NAME:  Qwen3-4B
  BATCH_SIZE:  32
  USE_LORA:    true
  CP_SIZE:     1
  VAL_INT:     62
  NUM_ROLL:    2500
  LR_WARMUP:   125
  RUN_NAME:    qwen4B_lora_bs_32_13068226
  RUN_ID:      13068226
  WAVE_NAME:   BROKEN_RESOLUTIONS
  MODEL_PATH:  /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
  SAVE_PATH:   /p/scratch/envcomp/mihir/huggingface/shared_data/13068226/checkpoints
============================================
WANDB_ARGS: --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen4B_lora_bs_32_13068226 --wandb-run-id 13068226 --wandb-dir /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
W0107 16:12:55.198000 4078750 torch/distributed/run.py:774] 
W0107 16:12:55.198000 4078750 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 4078750 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 16:12:55.198000 4078750 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 3489060 torch/distributed/run.py:774] 
W0107 16:12:55.198000 3489060 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 3489060 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 16:12:55.198000 3489060 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 3687384 torch/distributed/run.py:774] 
W0107 16:12:55.198000 3687384 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 3687384 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 16:12:55.198000 3687384 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 1451347 torch/distributed/run.py:774] 
W0107 16:12:55.198000 1451347 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 1451347 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 16:12:55.198000 1451347 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 3420207 torch/distributed/run.py:774] 
W0107 16:12:55.198000 3420207 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 3420207 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 16:12:55.198000 3420207 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 147367 torch/distributed/run.py:774] 
W0107 16:12:55.198000 147367 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 147367 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 16:12:55.198000 147367 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 107074 torch/distributed/run.py:774] 
W0107 16:12:55.198000 107074 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.198000 107074 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 16:12:55.198000 107074 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.199000 116438 torch/distributed/run.py:774] 
W0107 16:12:55.199000 116438 torch/distributed/run.py:774] *****************************************
W0107 16:12:55.199000 116438 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
W0107 16:12:55.199000 116438 torch/distributed/run.py:774] *****************************************
[W107 16:12:55.371673981 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:12:55.347609983 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:12:55.399683348 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:12:55.390757911 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:12:55.313923471 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:12:55.363993565 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:12:55.400228664 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:12:55.492147302 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:21] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:22] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:22] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:22] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:22] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:22] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:22] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[2026-01-07 16:14:22] arguments.py:1550 - n_samples_per_prompt is set to 1, grpo_std_normalization will be set to False.
[2026-01-07 16:14:22] arguments.py:1595 - args.eval_max_context_len is not set. Use args.rollout_max_context_len 4100 as default value.
[W107 16:14:22.824338956 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.824346626 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.739274914 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.825494647 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.825497457 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.740200608 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.826355758 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.827105858 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.826101149 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.826778536 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.803085166 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.804037800 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.832292647 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.833223860 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.805891566 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.824610783 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.806525809 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.747370772 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.825475092 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.748059965 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.832110162 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.835442857 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.832588657 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.836129559 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.832912150 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.833307424 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.808851827 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.809380209 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.810896544 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.811476656 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.802473605 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.839689091 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.803222320 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.803452701 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.840363593 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.831621471 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.831621491 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.839709088 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.803904034 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.754109553 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.832207947 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.754405844 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.840452359 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.832344268 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.832630461 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.754788056 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.755101928 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.833279107 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.806939403 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.807582818 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.811084440 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.811710594 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.851367617 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.852070069 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.851420744 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.851858918 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.813390245 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.814816865 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.814913486 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.816241546 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.830065235 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.830708440 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[W107 16:14:22.831185883 socket.cpp:755] [c10d] The client socket cannot be initialized to connect to [jwb0701.juwels]:29500 (errno: 97 - Address family not supported by protocol).
[W107 16:14:22.831805408 Utils.hpp:137] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
[Gloo] Rank 0 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 1 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 25 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 2 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 3 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 0] Distributed initialized: world_size=32, local_rank=0
[Gloo] Rank 4 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 28 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 29 is connected to 31[Gloo] Rank  peer ranks. Expected number of connected peer ranks is : 3131 is connected to 
31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 25] Distributed initialized: world_size=32, local_rank=1
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 1] Distributed initialized: world_size=32, local_rank=1
[Gloo] Rank 5 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 27 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 26 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 24 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 2] Distributed initialized: world_size=32, local_rank=2
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 3] Distributed initialized: world_size=32, local_rank=3
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 29] Distributed initialized: world_size=32, local_rank=1
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 28] Distributed initialized: world_size=32, local_rank=0
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 31] Distributed initialized: world_size=32, local_rank=3
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 26] Distributed initialized: world_size=32, local_rank=2
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 27] Distributed initialized: world_size=32, local_rank=3
[Gloo] Rank 6 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 24] Distributed initialized: world_size=32, local_rank=0
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 4] Distributed initialized: world_size=32, local_rank=0
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 5] Distributed initialized: world_size=32, local_rank=1
[Gloo] Rank 7 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 6] Distributed initialized: world_size=32, local_rank=2
[Gloo] Rank 30 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 7] Distributed initialized: world_size=32, local_rank=3
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 30] Distributed initialized: world_size=32, local_rank=2
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 26] Device mesh: dp_size=32, cp_size=1, dp_rank=26, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 24] Device mesh: dp_size=32, cp_size=1, dp_rank=24, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 27] Device mesh: dp_size=32, cp_size=1, dp_rank=27, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 25] Device mesh: dp_size=32, cp_size=1, dp_rank=25, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 0] Device mesh: dp_size=32, cp_size=1, dp_rank=0, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 2] Device mesh: dp_size=32, cp_size=1, dp_rank=2, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 1] Device mesh: dp_size=32, cp_size=1, dp_rank=1, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 3] Device mesh: dp_size=32, cp_size=1, dp_rank=3, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 28] Device mesh: dp_size=32, cp_size=1, dp_rank=28, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 29] Device mesh: dp_size=32, cp_size=1, dp_rank=29, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 7] Device mesh: dp_size=32, cp_size=1, dp_rank=7, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 5] Device mesh: dp_size=32, cp_size=1, dp_rank=5, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 6] Device mesh: dp_size=32, cp_size=1, dp_rank=6, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 31] Device mesh: dp_size=32, cp_size=1, dp_rank=31, cp_rank=0
[Gloo] Rank 8 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 9 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 8] Distributed initialized: world_size=32, local_rank=0
[Gloo] Rank 10 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 30] Device mesh: dp_size=32, cp_size=1, dp_rank=30, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 9] Distributed initialized: world_size=32, local_rank=1
[Gloo] Rank 11 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 10] Distributed initialized: world_size=32, local_rank=2
[Gloo] Rank 12 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 13 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 11] Distributed initialized: world_size=32, local_rank=3
[Gloo] Rank 14 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 15 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 12] Distributed initialized: world_size=32, local_rank=0
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 13] Distributed initialized: world_size=32, local_rank=1
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 14] Distributed initialized: world_size=32, local_rank=2
[Gloo] Rank 16 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 4] Device mesh: dp_size=32, cp_size=1, dp_rank=4, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 15] Distributed initialized: world_size=32, local_rank=3
[Gloo] Rank 17 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 18 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 19 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 20 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 16] Distributed initialized: world_size=32, local_rank=0
[Gloo] Rank 22 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[Gloo] Rank 21 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 17] Distributed initialized: world_size=32, local_rank=1
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 18] Distributed initialized: world_size=32, local_rank=2
[Gloo] Rank 23 is connected to 31 peer ranks. Expected number of connected peer ranks is : 31
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 19] Distributed initialized: world_size=32, local_rank=3
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 20] Distributed initialized: world_size=32, local_rank=0
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 21] Distributed initialized: world_size=32, local_rank=1
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 22] Distributed initialized: world_size=32, local_rank=2
[2026-01-07 16:14:22] train_sft.py:120 - [Rank 23] Distributed initialized: world_size=32, local_rank=3
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 10] Device mesh: dp_size=32, cp_size=1, dp_rank=10, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 8] Device mesh: dp_size=32, cp_size=1, dp_rank=8, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 11] Device mesh: dp_size=32, cp_size=1, dp_rank=11, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 9] Device mesh: dp_size=32, cp_size=1, dp_rank=9, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 15] Device mesh: dp_size=32, cp_size=1, dp_rank=15, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 13] Device mesh: dp_size=32, cp_size=1, dp_rank=13, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 12] Device mesh: dp_size=32, cp_size=1, dp_rank=12, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 14] Device mesh: dp_size=32, cp_size=1, dp_rank=14, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 18] Device mesh: dp_size=32, cp_size=1, dp_rank=18, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 17] Device mesh: dp_size=32, cp_size=1, dp_rank=17, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 20] Device mesh: dp_size=32, cp_size=1, dp_rank=20, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 23] Device mesh: dp_size=32, cp_size=1, dp_rank=23, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 21] Device mesh: dp_size=32, cp_size=1, dp_rank=21, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 22] Device mesh: dp_size=32, cp_size=1, dp_rank=22, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 16] Device mesh: dp_size=32, cp_size=1, dp_rank=16, cp_rank=0
[2026-01-07 16:14:22] train_sft.py:146 - [Rank 19] Device mesh: dp_size=32, cp_size=1, dp_rank=19, cp_rank=0
[2026-01-07 16:14:22] wandb_utils.py:72 - W&B logs will be stored in: /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS
wandb: Tracking run with wandb version 0.23.1
wandb: W&B syncing is set to `offline` in this directory. Run `wandb online` or set WANDB_MODE=online to enable cloud syncing.
wandb: Run data is saved locally in /p/scratch/envcomp/logs/BROKEN_RESOLUTIONS/wandb/offline-run-20260107_161423-81hkoo75
wandb: Detected [openai] in use.
wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
[2026-01-07 16:16:11] train_sft.py:218 - [Rank 4] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:12] train_sft.py:218 - [Rank 3] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:12] train_sft.py:218 - [Rank 7] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:12] train_sft.py:218 - [Rank 24] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:13] train_sft.py:218 - [Rank 18] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 28] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:13] train_sft.py:218 - [Rank 2] Data source initialized: dataset_size=97, num_rollout=2500

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:13] train_sft.py:218 - [Rank 15] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 6] Data source initialized: dataset_size=97, num_rollout=2500
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 1] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:13] train_sft.py:218 - [Rank 29] Data source initialized: dataset_size=97, num_rollout=2500

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 5] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 26] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 27] Data source initialized: dataset_size=97, num_rollout=2500

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 31] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 13] Data source initialized: dataset_size=97, num_rollout=2500
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 0] Data source initialized: dataset_size=97, num_rollout=2500

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 14] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:13] train_sft.py:218 - [Rank 12] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:13] train_sft.py:218 - [Rank 30] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 10] Data source initialized: dataset_size=97, num_rollout=2500
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 21] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 8] Data source initialized: dataset_size=97, num_rollout=2500

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:13] train_sft.py:218 - [Rank 11] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:14] train_sft.py:218 - [Rank 16] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:14] train_sft.py:218 - [Rank 17] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:14] train_sft.py:218 - [Rank 19] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:14] train_sft.py:218 - [Rank 22] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
[2026-01-07 16:16:14] train_sft.py:218 - [Rank 9] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:14] train_sft.py:218 - [Rank 25] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:15] train_sft.py:218 - [Rank 20] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s][2026-01-07 16:16:16] train_sft.py:218 - [Rank 23] Data source initialized: dataset_size=97, num_rollout=2500
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`

Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:08<00:17,  8.61s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:21, 10.52s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:08<00:17,  8.73s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:09<00:19,  9.73s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:21, 10.62s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:21, 10.73s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:20, 10.44s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.11s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:21, 10.77s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:08<00:16,  8.10s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:09<00:18,  9.10s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:21, 10.98s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.18s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:21, 10.56s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:12<00:24, 12.09s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:09<00:19,  9.69s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.05s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:09<00:19,  9.71s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.91s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.09s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.35s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.42s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.40s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:23, 11.72s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:21, 10.78s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:21, 10.68s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:12<00:24, 12.08s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:21, 10.86s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.48s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.36s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:11<00:22, 11.42s/it]
Loading checkpoint shards:  33%|███▎      | 1/3 [00:10<00:21, 10.54s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:24<00:12, 12.94s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:13, 13.30s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:23<00:12, 12.53s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:24<00:12, 12.59s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:23<00:12, 12.04s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:21<00:11, 11.03s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:22<00:11, 11.44s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:22<00:11, 11.66s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  7.02s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  8.12s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.25s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  7.04s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  8.15s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.47s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.44s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.75s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  6.51s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:23<00:00,  7.70s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  6.73s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:21<00:00,  6.18s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  8.01s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:21<00:00,  7.19s/it]


Loading checkpoint shards: 100%|██████████| 3/3 [00:22<00:00,  6.40s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:22<00:00,  7.53s/it]
[2026-01-07 16:16:37] train_sft.py:299 - [Rank 5] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:37] train_sft.py:299 - [Rank 4] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:37] train_sft.py:299 - [Rank 6] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:37] train_sft.py:299 - [Rank 7] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:37] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:37] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:37] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:37] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:37] train_sft.py:299 - [Rank 23] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:37] train_sft.py:299 - [Rank 21] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:37] train_sft.py:299 - [Rank 20] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:37] train_sft.py:299 - [Rank 22] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:37] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:37] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:37] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:37] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32

Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.46s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.97s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:13, 13.39s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.51s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.98s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.93s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.66s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.53s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.44s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.66s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.46s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.55s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:27<00:14, 14.09s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.75s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.47s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.61s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.50s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.83s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.46s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.52s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.86s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.76s/it]


Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.59s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.97s/it]
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 30] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 28] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 31] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 29] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32

Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.22s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.53s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.25s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.25s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.58s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.58s/it]


Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  7.53s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  9.00s/it]
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 18] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 19] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 17] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 16] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32

Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.50s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.54s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.49s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.61s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.97s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  7.65s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  7.83s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  9.32s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  9.04s/it]


Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.55s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.90s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.50s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.90s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.55s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.50s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.91s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.99s/it]


Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.48s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.89s/it]

Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:13, 13.15s/it][2026-01-07 16:16:40] train_sft.py:299 - [Rank 3] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 1] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 2] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 14] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 15] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 13] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 0] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 12] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32

Loading checkpoint shards:  67%|██████▋   | 2/3 [00:27<00:14, 14.29s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:27<00:13, 13.96s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:13, 13.40s/it]
Loading checkpoint shards:  67%|██████▋   | 2/3 [00:27<00:13, 13.96s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.51s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.91s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  7.30s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.53s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:25<00:00,  8.62s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.93s/it]


Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.54s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.96s/it]
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 10] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 8] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 11] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] train_sft.py:299 - [Rank 9] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:40] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32

Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  7.77s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  7.75s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  7.47s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  9.15s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:27<00:00,  9.14s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:26<00:00,  8.70s/it]

Loading checkpoint shards: 100%|██████████| 3/3 [00:28<00:00,  7.96s/it]
Loading checkpoint shards: 100%|██████████| 3/3 [00:28<00:00,  9.43s/it]
[2026-01-07 16:16:41] train_sft.py:299 - [Rank 24] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:41] train_sft.py:299 - [Rank 26] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:41] train_sft.py:299 - [Rank 27] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:41] train_sft.py:299 - [Rank 25] Applied LoRA: LoRAConfig(lora_rank=8, lora_alpha=16, lora_dropout=0.0, target_modules=['q_proj', 'v_proj'], bias='none')
[2026-01-07 16:16:41] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:41] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:41] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:41] actor.py:1103 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 20] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 20] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 10] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 21] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 10] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 21] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 19] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 18] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 22] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 19] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 18] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 22] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 13] Model initialized with FSDP
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 24] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 25] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 13] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 24] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 25] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 27] Model initialized with FSDP
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 16] Model initialized with FSDP
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 27] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 16] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 9] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 9] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 23] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 23] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 8] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 8] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 26] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 17] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 26] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 29] Model initialized with FSDP
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 17] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 12] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 14] Model initialized with FSDP
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 29] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 15] Model initialized with FSDP
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 12] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 14] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 15] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 31] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 31] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 28] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 28] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 11] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 11] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 30] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 30] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 4] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 4] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 7] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 5] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 7] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 5] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 6] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 6] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 3] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 1] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:321 - [Rank 2] Model initialized with FSDP
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 3] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 1] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] train_sft.py:330 - [Rank 2] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
[2026-01-07 16:16:47] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:49] train_sft.py:321 - [Rank 0] Model initialized with FSDP
[2026-01-07 16:16:49] train_sft.py:330 - [Rank 0] LoRA: 2,949,120 trainable params out of 4,025,417,216 total (0.07%)
[2026-01-07 16:16:49] lr_scheduler.py:80 - > learning rate decay style: WSD
[2026-01-07 16:16:49] checkpoint.py:121 - [FSDP] No tracker file at /p/scratch/envcomp/mihir/huggingface/Qwen3-4B/latest_checkpointed_iteration.txt; skipping load.
/p/project1/envcomp/mahajan1/miles/.venv/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
  warnings.warn(  # warn only once
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 3] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 0] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 1] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 2] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 21] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 22] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 20] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 23] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 3] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 12] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 13] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 14] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 10] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 11] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 9] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 8] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 7] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 4] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 5] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 28] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 29] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 26] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 27] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 25] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 1] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 2] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 17] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 18] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 12] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 15] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 13] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 14] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 4] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 5] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 7] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 6] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 31] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 0] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 16] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 15] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 10] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 11] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 6] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 28] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 21] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 22] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 20] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 27] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 25] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 26] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 17] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 19] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 8] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 9] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 29] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 23] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 18] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 30] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 16] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 31] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 19] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 30] Starting training: rollout_id 0 -> 2500
[2026-01-07 16:16:49] train_sft.py:102 - [Rank 24] SFTTrainer initialized successfully
[2026-01-07 16:16:49] train_sft.py:701 - [Rank 24] Starting training: rollout_id 0 -> 2500

actor_val:   0%|          | 0/100 [00:00<?, ?it/s][2026-01-07 16:16:49] train_sft.py:375 - SFT rollout sample: tokens_len=3598, response_length=960
[2026-01-07 16:16:50] train_sft.py:637 - [Rank 26] No batches to validate on rollout 0, validation step 0
[2026-01-07 16:16:50] train_sft.py:637 - [Rank 30] No batches to validate on rollout 0, validation step 0
[2026-01-07 16:16:50] train_sft.py:637 - [Rank 24] No batches to validate on rollout 0, validation step 0
[2026-01-07 16:16:50] train_sft.py:637 - [Rank 31] No batches to validate on rollout 0, validation step 0
[2026-01-07 16:16:50] train_sft.py:637 - [Rank 29] No batches to validate on rollout 0, validation step 0
[2026-01-07 16:16:50] train_sft.py:637 - [Rank 28] No batches to validate on rollout 0, validation step 0
[2026-01-07 16:16:50] train_sft.py:637 - [Rank 27] No batches to validate on rollout 0, validation step 0
[2026-01-07 16:16:50] train_sft.py:637 - [Rank 25] No batches to validate on rollout 0, validation step 0

actor_val:   1%|          | 1/100 [00:06<10:30,  6.37s/it][2026-01-07 16:16:56] train_sft.py:375 - SFT rollout sample: tokens_len=3806, response_length=2853
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 20] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 16] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 17] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 18] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 19] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 13] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 15] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 14] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 12] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 23] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 21] No batches to validate on rollout 0, validation step 1
[2026-01-07 16:16:56] train_sft.py:637 - [Rank 22] No batches to validate on rollout 0, validation step 1