[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/run_qwen3-0.6b-sft-torch_36286.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-sft-torch


trap 'echo "Job killed by signal"' USR1

cat $0

source ~/.bashrc

micromamba activate miles

srun sh scripts/run-qwen3-0.6B-torch-sft.shRunning inside SLURM, Job ID 36286.
2025-12-11 19:39:14,308	INFO scripts.py:1371 -- Did not find any active Ray processes.
HAS_NVLINK: 0 (detected 0 NVLink references)
Detected Head Node IP: 10.86.2.80
2025-12-11 19:39:21,159	INFO usage_lib.py:447 -- Usage stats collection is disabled.
2025-12-11 19:39:21,159	INFO scripts.py:919 -- Local node IP: 10.86.2.80
2025-12-11 19:39:28,300	SUCC scripts.py:963 -- --------------------
2025-12-11 19:39:28,300	SUCC scripts.py:964 -- Ray runtime started.
2025-12-11 19:39:28,300	SUCC scripts.py:965 -- --------------------
2025-12-11 19:39:28,300	INFO scripts.py:967 -- Next steps
2025-12-11 19:39:28,300	INFO scripts.py:970 -- To add another node to this Ray cluster, run
2025-12-11 19:39:28,300	INFO scripts.py:973 --   ray start --address='10.86.2.80:6379'
2025-12-11 19:39:28,301	INFO scripts.py:982 -- To connect to this Ray cluster:
2025-12-11 19:39:28,301	INFO scripts.py:984 -- import ray
2025-12-11 19:39:28,301	INFO scripts.py:985 -- ray.init(_node_ip_address='10.86.2.80')
2025-12-11 19:39:28,301	INFO scripts.py:997 -- To submit a Ray job using the Ray Jobs CLI:
2025-12-11 19:39:28,301	INFO scripts.py:998 --   RAY_API_SERVER_ADDRESS='http://10.86.2.80:8265' ray job submit --working-dir . -- python my_script.py
2025-12-11 19:39:28,301	INFO scripts.py:1007 -- See https://docs.ray.io/en/latest/cluster/running-applications/job-submission/index.html 
2025-12-11 19:39:28,301	INFO scripts.py:1011 -- for more information on submitting Ray jobs to the Ray cluster.
2025-12-11 19:39:28,301	INFO scripts.py:1016 -- To terminate the Ray runtime, run
2025-12-11 19:39:28,301	INFO scripts.py:1017 --   ray stop
2025-12-11 19:39:28,301	INFO scripts.py:1020 -- To view the status of the cluster, use
2025-12-11 19:39:28,301	INFO scripts.py:1021 --   ray status
2025-12-11 19:39:28,301	INFO scripts.py:1025 -- To monitor and debug Ray, view the dashboard at 
2025-12-11 19:39:28,301	INFO scripts.py:1026 --   10.86.2.80:8265
2025-12-11 19:39:28,301	INFO scripts.py:1033 -- If connection to the dashboard fails, check your firewall settings and network configuration.
Ray started. Waiting for Dashboard to be ready...
Dashboard is up!
2025-12-11 19:39:37,371	INFO cli.py:41 -- Job submission server address: http://10.86.2.80:8265
2025-12-11 19:39:43,600	SUCC cli.py:65 -- -------------------------------------------------------
2025-12-11 19:39:43,600	SUCC cli.py:66 -- Job 'raysubmit_4DsLVRzyAmBnNWKJ' submitted successfully
2025-12-11 19:39:43,600	SUCC cli.py:67 -- -------------------------------------------------------
2025-12-11 19:39:43,601	INFO cli.py:291 -- Next steps
2025-12-11 19:39:43,601	INFO cli.py:292 -- Query the logs of the job:
2025-12-11 19:39:43,601	INFO cli.py:294 -- ray job logs raysubmit_4DsLVRzyAmBnNWKJ
2025-12-11 19:39:43,601	INFO cli.py:296 -- Query the status of the job:
2025-12-11 19:39:43,601	INFO cli.py:298 -- ray job status raysubmit_4DsLVRzyAmBnNWKJ
2025-12-11 19:39:43,601	INFO cli.py:300 -- Request the job to be stopped:
2025-12-11 19:39:43,601	INFO cli.py:302 -- ray job stop raysubmit_4DsLVRzyAmBnNWKJ
2025-12-11 19:39:43,604	INFO cli.py:312 -- Tailing logs until the job exits (disable with --no-wait):
2025-12-11 19:39:43,280	INFO job_manager.py:568 -- Runtime env is setting up.
Running entrypoint for job raysubmit_4DsLVRzyAmBnNWKJ: python3 train.py --hf-checkpoint /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --ref-load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --rollout-function-path miles.rollout.sft_rollout.generate_rollout --prompt-data /fast/project/HFMI_SynergyUnit/tab_model/huggingface/openhermes2_5.parquet --input-key messages --apply-chat-template --rollout-shuffle --num-epoch 3 --rollout-batch-size 128 --global-batch-size 128 --loss-type sft_loss --calculate-per-token-loss --disable-compute-advantages-and-returns --debug-train-only --optimizer adam --lr 1e-5 --lr-decay-style constant --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.98 --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen3-0.6b-sft-torch --train-backend fsdp --update-weight-buffer-size 536870912 --gradient-checkpointing --attn-implementation flash_attention_3 --train-env-vars {\"PYTORCH_CUDA_ALLOC_CONF\":\"expandable_segments:True\"} --actor-num-gpus-per-node 2 --use-dynamic-batch-size --max-tokens-per-gpu 9216
/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
[2025-12-11 19:40:04] INFO trace.py:52: opentelemetry package is not installed, tracing disabled
DEBUG: Configured logger.
2025-12-11 19:40:09,349	INFO worker.py:1696 -- Using address 10.86.2.80:6379 set in the environment variable RAY_ADDRESS
2025-12-11 19:40:09,361	INFO worker.py:1837 -- Connecting to existing Ray cluster at address: 10.86.2.80:6379...
2025-12-11 19:40:09,375	INFO worker.py:2014 -- Connected to Ray cluster. View the dashboard at http://10.86.2.80:8265 
/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py:2062: FutureWarning: Tip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0
  warnings.warn(
DEBUG: Created placement groups.
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: WARNING The `shared` mode feature is experimental and may change. Please contact support@wandb.com for guidance and to report any issues.
wandb: setting up run 0cdor63m
wandb: Tracking run with wandb version 0.23.1
wandb: Run data is saved locally in /fast/home/mihir.mahajan/Projects/miles/wandb/run-20251211_194013-0cdor63m
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run qwen3-0.6b-sft-torch_526x703k-RANK_0
wandb: ⭐️ View project at https://wandb.ai/instant-uv/crowd-pilot-miles
wandb: 🚀 View run at https://wandb.ai/instant-uv/crowd-pilot-miles/runs/0cdor63m
wandb: Detected [openai] in use.
wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
DEBUG: Initialized tracking.
DEBUG: Created rollout manager.
DEBUG: allocating actor_model
(pid=721547) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
(pid=721547)   import pynvml  # type: ignore[import]
(RolloutManager pid=721547) DEBUG: configure_logger() executed
(RolloutManager pid=721547) DEBUG: self.args = args executed
(RolloutManager pid=721547) DEBUG: self.pg = pg executed
(RolloutManager pid=721547) DEBUG: data_source_cls loaded
DEBUG: allocated actor_model
DEBUG: critic_model is None
(pid=722188) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. [repeated 2x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)
(pid=722188)   import pynvml  # type: ignore[import] [repeated 2x across cluster]
(FSDPTrainRayActor pid=721548) [Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
(FSDPTrainRayActor pid=721548) [2025-12-11 19:40:49] train_actor.py:96 - Set NUMA affinity for GPU 0
(FSDPTrainRayActor pid=721548) [2025-12-11 19:40:49] actor.py:217 - [Rank 0] Device mesh (2D): world_size=2, cp_size=1, dp_size=2
(FSDPTrainRayActor pid=721548) [2025-12-11 19:40:49] actor.py:221 - [Rank 0] Mesh shape: (2, 1), dp_rank=0, cp_rank=0
(FSDPTrainRayActor pid=721548) [2025-12-11 19:40:49] actor.py:228 - [Rank 0] Pure DP mode (cp_size=1)
(FSDPTrainRayActor pid=721548) wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
(FSDPTrainRayActor pid=721548) wandb: WARNING The `shared` mode feature is experimental and may change. Please contact support@wandb.com for guidance and to report any issues.
(FSDPTrainRayActor pid=721548) wandb: WARNING Using a boolean value for 'reinit' is deprecated. Use 'return_previous' or 'finish_previous' instead.
(FSDPTrainRayActor pid=721548) wandb: setting up run 0cdor63m
(FSDPTrainRayActor pid=721548) wandb: Tracking run with wandb version 0.23.1
(FSDPTrainRayActor pid=721548) wandb: Run data is saved locally in /fast/home/mihir.mahajan/Projects/miles/wandb/run-20251211_194050-0cdor63m
(FSDPTrainRayActor pid=721548) wandb: Run `wandb offline` to turn off syncing.
(FSDPTrainRayActor pid=721548) wandb: Resuming run qwen3-0.6b-sft-torch_526x703k-RANK_0
(FSDPTrainRayActor pid=721548) wandb: ⭐️ View project at https://wandb.ai/instant-uv/crowd-pilot-miles
(FSDPTrainRayActor pid=721548) wandb: 🚀 View run at https://wandb.ai/instant-uv/crowd-pilot-miles/runs/0cdor63m
(FSDPTrainRayActor pid=721548) [2025-12-11 19:40:52] actor.py:252 - [Rank 0] tie_word_embeddings=True, loading full model to CPU on all ranks
(FSDPTrainRayActor pid=721548) Flash Attention 3 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("meta-llama/Llama-3.2-1B", attn_implementation="flash_attention_3", dtype=torch.float16)`
(FSDPTrainRayActor pid=721548) Flash Attention 3 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("meta-llama/Llama-3.2-1B", attn_implementation="flash_attention_3", dtype=torch.float16)`
(FSDPTrainRayActor pid=722188) [2025-12-11 19:40:53] actor.py:1081 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
(FSDPTrainRayActor pid=721548) hai008:721548:721548 [0] NCCL INFO Bootstrap: Using bond0:10.86.2.80<0>
(FSDPTrainRayActor pid=721548) hai008:721548:721548 [0] NCCL INFO cudaDriverVersion 12080
(FSDPTrainRayActor pid=721548) hai008:721548:721548 [0] NCCL INFO NCCL version 2.27.3+cuda12.9
(FSDPTrainRayActor pid=721548) hai008:721548:721548 [0] NCCL INFO Comm config Blocking set to 1
(FSDPTrainRayActor pid=722188) [Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO NET/Plugin: Could not find: libnccl-net.so. 
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO NCCL_IB_DISABLE set by environment to 1.
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO NET/Socket : Using [0]bond0:10.86.2.80<0>
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Initialized NET plugin Socket
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Assigned NET plugin Socket to comm
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Using network Socket
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO ncclCommInitRankConfig comm 0x559d81829750 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 19000 commId 0xdde15a968db157a - Init START
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO RAS client listening socket at ::1<28028>
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Bootstrap timings total 0.035764 (create 0.000028, send 0.000114, recv 0.035165, ring 0.000032, delay 0.000001)
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0.
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO MNNVL busId 0x19000 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO NCCL_P2P_DISABLE set by environment to 1
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Setting affinity for GPU 0 to 0,24,112,136
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0.
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO comm 0x559d81829750 rank 0 nRanks 2 nNodes 1 localRanks 2 localRank 0 MNNVL 0
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Channel 00/04 : 0 1
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Channel 01/04 : 0 1
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Channel 02/04 : 0 1
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Channel 03/04 : 0 1
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] -1/-1/-1->0->1 [2] 1/-1/-1->0->-1 [3] -1/-1/-1->0->1
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO P2P Chunksize set to 131072
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. 
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Check P2P Type isAllDirectP2p 0 directMode 0
(FSDPTrainRayActor pid=721548) hai008:721548:722480 [0] NCCL INFO [Proxy Service] Device 0 CPU core 136
(FSDPTrainRayActor pid=721548) hai008:721548:722481 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 136
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Channel 00 : 0[0] -> 1[1] via SHM/direct/direct
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Channel 01 : 0[0] -> 1[1] via SHM/direct/direct
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Channel 02 : 0[0] -> 1[1] via SHM/direct/direct
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Channel 03 : 0[0] -> 1[1] via SHM/direct/direct
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Connected all trees
(FSDPTrainRayActor pid=721548) hai008:721548:722485 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 136
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 2 p2p channels per peer
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO CC Off, workFifoBytes 1048576
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so. Using internal tuner plugin.
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO ncclCommInitRankConfig comm 0x559d81829750 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 19000 commId 0xdde15a968db157a - Init COMPLETE
(FSDPTrainRayActor pid=721548) hai008:721548:722475 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 2 total 0.41 (kernels 0.23, alloc 0.02, bootstrap 0.04, allgathers 0.00, topo 0.02, graphs 0.00, connections 0.09, rest 0.01)
(FSDPTrainRayActor pid=722188) [2025-12-11 19:40:55] lr_scheduler.py:80 - > learning rate decay style: constant
(FSDPTrainRayActor pid=722188) [2025-12-11 19:40:55] checkpoint.py:97 - [FSDP] No tracker file at /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B/latest_checkpointed_iteration.txt; skipping load.
(FSDPTrainRayActor pid=722188) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
(FSDPTrainRayActor pid=722188)   warnings.warn(  # warn only once
(FSDPTrainRayActor pid=722188) [2025-12-11 19:40:49] train_actor.py:96 - Set NUMA affinity for GPU 1
(FSDPTrainRayActor pid=722188) [2025-12-11 19:40:49] actor.py:217 - [Rank 1] Device mesh (2D): world_size=2, cp_size=1, dp_size=2
(FSDPTrainRayActor pid=722188) [2025-12-11 19:40:49] actor.py:221 - [Rank 1] Mesh shape: (2, 1), dp_rank=1, cp_rank=0
(FSDPTrainRayActor pid=722188) [2025-12-11 19:40:49] actor.py:228 - [Rank 1] Pure DP mode (cp_size=1)
DEBUG: got start_rollout_ids
DEBUG: asserted start_rollout_ids length
(FSDPTrainRayActor pid=721548) [2025-12-11 19:40:55] timer.py:24 - Timer train_wait start
(FSDPTrainRayActor pid=722188) set_rollout_manager
(FSDPTrainRayActor pid=722188) set_rollout_manager done
(FSDPTrainRayActor pid=721548) set_train_parallel_config
DEBUG: set rollout_manager for actor_model
DEBUG: loaded rollout_manager data
DEBUG: returning actor_model and critic_model
DEBUG: Updated actor model weights.
DEBUG: Defined offload_train function.
DEBUG: Defined onload_rollout function.
Starting rollout 0.
(RolloutManager pid=721547) DEBUG: self.data_source initialized
(RolloutManager pid=721547) DEBUG: self.generate_rollout loaded
(RolloutManager pid=721547) DEBUG: self.eval_generate_rollout loaded
(RolloutManager pid=721547) DEBUG: self.custom_reward_post_process_func initialized to None
(RolloutManager pid=721547) DEBUG: logger.info for generate_rollout executed
(RolloutManager pid=721547) DEBUG: logger.info for eval_generate_rollout executed
(RolloutManager pid=721547) DEBUG: self.all_rollout_engines initialized as empty list (debug_train_only)
(RolloutManager pid=721547) DEBUG: self.num_new_engines initialized
(RolloutManager pid=721547) DEBUG: self.nodes_per_engine calculated
(RolloutManager pid=721547) DEBUG: self.rollout_engine_lock initialized
(RolloutManager pid=721547) DEBUG: self._metric_checker initialized
(RolloutManager pid=721547) set_train_parallel_config done
(RolloutManager pid=721547) DEBUG: generate method started
(RolloutManager pid=721547) DEBUG: Calling _get_rollout_data
(FSDPTrainRayActor pid=722188) hai008:722188:722188 [1] NCCL INFO Bootstrap: Using bond0:10.86.2.80<0>
(FSDPTrainRayActor pid=722188) hai008:722188:722188 [1] NCCL INFO cudaDriverVersion 12080
(FSDPTrainRayActor pid=722188) hai008:722188:722188 [1] NCCL INFO NCCL version 2.27.3+cuda12.9
(FSDPTrainRayActor pid=722188) hai008:722188:722188 [1] NCCL INFO Comm config Blocking set to 1
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO NET/Plugin: Could not find: libnccl-net.so. 
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO NCCL_IB_DISABLE set by environment to 1.
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO NET/Socket : Using [0]bond0:10.86.2.80<0>
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO Initialized NET plugin Socket
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO Assigned NET plugin Socket to comm
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO Using network Socket
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO ncclCommInitRankConfig comm 0x564baea1c470 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 3b000 commId 0xdde15a968db157a - Init START
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO RAS client listening socket at ::1<28028>
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO Bootstrap timings total 0.000751 (create 0.000026, send 0.000112, recv 0.000187, ring 0.000053, delay 0.000000)
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0.
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO MNNVL busId 0x3b000 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO NCCL_P2P_DISABLE set by environment to 1
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO Setting affinity for GPU 1 to 0,24,112,136
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0.
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO comm 0x564baea1c470 rank 1 nRanks 2 nNodes 1 localRanks 2 localRank 1 MNNVL 0
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] 0/-1/-1->1->-1 [2] -1/-1/-1->1->0 [3] 0/-1/-1->1->-1
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO P2P Chunksize set to 131072
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. 
(FSDPTrainRayActor pid=722188) hai008:722188:722482 [1] NCCL INFO [Proxy Service] Device 1 CPU core 136
(FSDPTrainRayActor pid=722188) hai008:722188:722483 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 136
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO Channel 03 : 1[1] -> 0[0] via SHM/direct/direct [repeated 4x across cluster]
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO Connected all trees
(FSDPTrainRayActor pid=722188) hai008:722188:722484 [1] NCCL INFO [Proxy Progress] Device 1 CPU core 112
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 2 p2p channels per peer
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so. Using internal tuner plugin.
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO ncclCommInitRankConfig comm 0x564baea1c470 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 3b000 commId 0xdde15a968db157a - Init COMPLETE
(FSDPTrainRayActor pid=722188) hai008:722188:722476 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 2 total 0.41 (kernels 0.27, alloc 0.01, bootstrap 0.00, allgathers 0.00, topo 0.02, graphs 0.00, connections 0.09, rest 0.02)
(FSDPTrainRayActor pid=721548) set_rollout_manager
(FSDPTrainRayActor pid=721548) set_rollout_manager done
(RolloutManager pid=721547) set_train_parallel_config
(RolloutManager pid=721547) [2025-12-11 19:59:19] rollout.py:74 - import miles.rollout.sft_rollout.generate_rollout as generate_rollout function.
(RolloutManager pid=721547) [2025-12-11 19:59:19] rollout.py:76 - import miles.rollout.sft_rollout.generate_rollout as eval_generate_rollout function.
(RolloutManager pid=721547) [2025-12-11 19:59:19] data_source.py:141 - Checkpoint /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B/rollout/global_dataset_state_dict_-1.pt does not exist.
(FSDPTrainRayActor pid=722188) [2025-12-11 19:40:52] actor.py:252 - [Rank 1] tie_word_embeddings=True, loading full model to CPU on all ranks
(FSDPTrainRayActor pid=722188) Flash Attention 3 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("meta-llama/Llama-3.2-1B", attn_implementation="flash_attention_3", dtype=torch.float16)` [repeated 2x across cluster]
(FSDPTrainRayActor pid=721548) [2025-12-11 19:40:54] actor.py:1081 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
(FSDPTrainRayActor pid=721548) [2025-12-11 19:40:55] lr_scheduler.py:80 - > learning rate decay style: constant
(FSDPTrainRayActor pid=721548) [2025-12-11 19:40:55] checkpoint.py:97 - [FSDP] No tracker file at /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B/latest_checkpointed_iteration.txt; skipping load.
(FSDPTrainRayActor pid=721548) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
(FSDPTrainRayActor pid=721548)   warnings.warn(  # warn only once
(FSDPTrainRayActor pid=721548) [2025-12-11 19:59:19] timer.py:24 - Timer update_weights start
(FSDPTrainRayActor pid=721548) [2025-12-11 19:59:19] timer.py:32 - Timer update_weights end (elapsed: 0.0s)
(RolloutManager pid=721547) [2025-12-11 19:59:20] sft_rollout.py:53 - sft_rollout::generate_rollout example data: sample=Sample(group_index=0, index=0, prompt=[{'content': "How can I sort a list of tuples in ascending order based on the second item in the tuple, the integer value? For example, I have a list like this `[('abc', 121),('abc', 231),('abc', 148), ('abc',221)]` and I want to sort it based on the integer values.", 'role': 'user'}, {'content': "Yes, it is possible to sort a list of tuples based on the second item in the tuples. Python provides a built-in function `sorted()` for this purpose. You can use a lambda function as the key which will be used to sort the list. In the lambda function, select the second item of the tuple (index 1) to sort on that.

Here is the code to do this:

```python
# your list of tuples
tuples_list = [('abc', 121),('abc', 231),('abc', 148), ('abc',221)]

# use sorted function
sorted_tuples_list = sorted(tuples_list, key = lambda x: x[1])

# print the sorted list
print(sorted_tuples_list)
```

In this code, `sorted()` is a built-in Python function that takes an iterable and returns a new sorted list of the elements in the iterable. The `key` parameter is a function that takes an element from the iterable and returns a value that will be used for sorting. In this case, we're using a lambda function that takes a tuple and returns the second element of the tuple (i.e., `x[1]`), so the list will be sorted based on the second elements of the tuples.

When you run this code, it will print:

```python
[('abc', 121), ('abc', 148), ('abc', 221), ('abc', 231)]
```

This is your original list, but sorted in ascending order based on the second item of the tuples.", 'role': 'assistant'}], tokens=[151644, 872, 198, 4340, 646, 358, 3378, 264, 1140, 315, 45225, 304, 35388, 1973, 3118, 389, 279, 2086, 1509, 304, 279, 14405, 11, 279, 7546, 897, 30, 1752, 3110, 11, 358, 614, 264, 1140, 1075, 419, 77644, 492, 13683, 516, 220, 16, 17, 16, 701, 492, 13683, 516, 220, 17, 18, 16, 701, 492, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 17, 17, 16, 7252, 63, 323, 358, 1366, 311, 3378, 432, 3118, 389, 279, 7546, 2750, 13, 151645, 198, 151644, 77091, 198, 9454, 11, 432, 374, 3204, 311, 3378, 264, 1140, 315, 45225, 3118, 389, 279, 2086, 1509, 304, 279, 45225, 13, 13027, 5707, 264, 5798, 3419, 729, 1565, 28298, 54258, 369, 419, 7428, 13, 1446, 646, 990, 264, 12459, 729, 438, 279, 1376, 892, 686, 387, 1483, 311, 3378, 279, 1140, 13, 758, 279, 12459, 729, 11, 3293, 279, 2086, 1509, 315, 279, 14405, 320, 1252, 220, 16, 8, 311, 3378, 389, 429, 382, 8420, 374, 279, 2038, 311, 653, 419, 1447, 73594, 12669, 198, 2, 697, 1140, 315, 45225, 198, 83, 28323, 2019, 284, 51128, 13683, 516, 220, 16, 17, 16, 701, 492, 13683, 516, 220, 17, 18, 16, 701, 492, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 17, 17, 16, 27771, 2, 990, 10615, 729, 198, 28298, 89269, 2019, 284, 10615, 1155, 28323, 2019, 11, 1376, 284, 12459, 856, 25, 856, 58, 16, 9604, 2, 1173, 279, 10615, 1140, 198, 1350, 44386, 89269, 2019, 340, 13874, 19324, 641, 419, 2038, 11, 1565, 28298, 54258, 374, 264, 5798, 3419, 13027, 729, 429, 4990, 458, 50834, 323, 4675, 264, 501, 10615, 1140, 315, 279, 5424, 304, 279, 50834, 13, 576, 1565, 792, 63, 5733, 374, 264, 729, 429, 4990, 458, 2392, 504, 279, 50834, 323, 4675, 264, 897, 429, 686, 387, 1483, 369, 28273, 13, 758, 419, 1142, 11, 582, 2299, 1667, 264, 12459, 729, 429, 4990, 264, 14405, 323, 4675, 279, 2086, 2392, 315, 279, 14405, 320, 72, 1734, 2572, 1565, 87, 58, 16, 60, 63, 701, 773, 279, 1140, 686, 387, 10615, 3118, 389, 279, 2086, 5424, 315, 279, 45225, 382, 4498, 498, 1598, 419, 2038, 11, 432, 686, 1173, 1447, 73594, 12669, 198, 58, 492, 13683, 516, 220, 16, 17, 16, 701, 4319, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 220, 17, 17, 16, 701, 4319, 13683, 516, 220, 17, 18, 16, 5563, 13874, 19324, 1986, 374, 697, 4024, 1140, 11, 714, 10615, 304, 35388, 1973, 3118, 389, 279, 2086, 1509, 315, 279, 45225, 13, 151645, 198], multimodal_inputs=None, response='', response_length=335, label=None, reward=0, loss_mask=[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], weight_versions=[], rollout_log_probs=None, rollout_routed_experts=None, remove_sample=False, status=<Status.PENDING: 'pending'>, metadata={}, train_metadata=None, spec_info=<miles.utils.types.Sample.SpecInfo object at 0x7f22ece6f170>) (raw)messages=[{'content': "How can I sort a list of tuples in ascending order based on the second item in the tuple, the integer value? For example, I have a list like this `[('abc', 121),('abc', 231),('abc', 148), ('abc',221)]` and I want to sort it based on the integer values.", 'role': 'user'}, {'content': "Yes, it is possible to sort a list of tuples based on the second item in the tuples. Python provides a built-in function `sorted()` for this purpose. You can use a lambda function as the key which will be used to sort the list. In the lambda function, select the second item of the tuple (index 1) to sort on that.

Here is the code to do this:

```python
# your list of tuples
tuples_list = [('abc', 121),('abc', 231),('abc', 148), ('abc',221)]

# use sorted function
sorted_tuples_list = sorted(tuples_list, key = lambda x: x[1])

# print the sorted list
print(sorted_tuples_list)
```

In this code, `sorted()` is a built-in Python function that takes an iterable and returns a new sorted list of the elements in the iterable. The `key` parameter is a function that takes an element from the iterable and returns a value that will be used for sorting. In this case, we're using a lambda function that takes a tuple and returns the second element of the tuple (i.e., `x[1]`), so the list will be sorted based on the second elements of the tuples.

When you run this code, it will print:

```python
[('abc', 121), ('abc', 148), ('abc', 221), ('abc', 231)]
```

This is your original list, but sorted in ascending order based on the second item of the tuples.", 'role': 'assistant'}] (raw)token_ids=[151644, 872, 198, 4340, 646, 358, 3378, 264, 1140, 315, 45225, 304, 35388, 1973, 3118, 389, 279, 2086, 1509, 304, 279, 14405, 11, 279, 7546, 897, 30, 1752, 3110, 11, 358, 614, 264, 1140, 1075, 419, 77644, 492, 13683, 516, 220, 16, 17, 16, 701, 492, 13683, 516, 220, 17, 18, 16, 701, 492, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 17, 17, 16, 7252, 63, 323, 358, 1366, 311, 3378, 432, 3118, 389, 279, 7546, 2750, 13, 151645, 198, 151644, 77091, 198, 9454, 11, 432, 374, 3204, 311, 3378, 264, 1140, 315, 45225, 3118, 389, 279, 2086, 1509, 304, 279, 45225, 13, 13027, 5707, 264, 5798, 3419, 729, 1565, 28298, 54258, 369, 419, 7428, 13, 1446, 646, 990, 264, 12459, 729, 438, 279, 1376, 892, 686, 387, 1483, 311, 3378, 279, 1140, 13, 758, 279, 12459, 729, 11, 3293, 279, 2086, 1509, 315, 279, 14405, 320, 1252, 220, 16, 8, 311, 3378, 389, 429, 382, 8420, 374, 279, 2038, 311, 653, 419, 1447, 73594, 12669, 198, 2, 697, 1140, 315, 45225, 198, 83, 28323, 2019, 284, 51128, 13683, 516, 220, 16, 17, 16, 701, 492, 13683, 516, 220, 17, 18, 16, 701, 492, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 17, 17, 16, 27771, 2, 990, 10615, 729, 198, 28298, 89269, 2019, 284, 10615, 1155, 28323, 2019, 11, 1376, 284, 12459, 856, 25, 856, 58, 16, 9604, 2, 1173, 279, 10615, 1140, 198, 1350, 44386, 89269, 2019, 340, 13874, 19324, 641, 419, 2038, 11, 1565, 28298, 54258, 374, 264, 5798, 3419, 13027, 729, 429, 4990, 458, 50834, 323, 4675, 264, 501, 10615, 1140, 315, 279, 5424, 304, 279, 50834, 13, 576, 1565, 792, 63, 5733, 374, 264, 729, 429, 4990, 458, 2392, 504, 279, 50834, 323, 4675, 264, 897, 429, 686, 387, 1483, 369, 28273, 13, 758, 419, 1142, 11, 582, 2299, 1667, 264, 12459, 729, 429, 4990, 264, 14405, 323, 4675, 279, 2086, 2392, 315, 279, 14405, 320, 72, 1734, 2572, 1565, 87, 58, 16, 60, 63, 701, 773, 279, 1140, 686, 387, 10615, 3118, 389, 279, 2086, 5424, 315, 279, 45225, 382, 4498, 498, 1598, 419, 2038, 11, 432, 686, 1173, 1447, 73594, 12669, 198, 58, 492, 13683, 516, 220, 16, 17, 16, 701, 4319, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 220, 17, 17, 16, 701, 4319, 13683, 516, 220, 17, 18, 16, 5563, 13874, 19324, 1986, 374, 697, 4024, 1140, 11, 714, 10615, 304, 35388, 1973, 3118, 389, 279, 2086, 1509, 315, 279, 45225, 13, 151645, 198] (raw)loss_mask=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] response_length=335
wandb: ERROR ray::RolloutManager.generate() (pid=721547, ip=10.86.2.80, actor_id=e975bf23916eb598b5f5e00c02000000, repr=<miles.ray.rollout.RolloutManager object at 0x7f2468350620>)
wandb: ERROR            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
wandb: ERROR            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
wandb: ERROR   File "/fast/home/mihir.mahajan/Projects/miles/miles/ray/rollout.py", line 131, in generate
wandb: ERROR     _log_rollout_data(rollout_id, self.args, data, metrics, time.time() - start_time)
wandb: ERROR   File "/fast/home/mihir.mahajan/Projects/miles/miles/ray/rollout.py", line 643, in _log_rollout_data
wandb: ERROR     tracking_utils.log(args, log_dict, step_key="rollout/step")
wandb: ERROR   File "/fast/home/mihir.mahajan/Projects/miles/miles/utils/tracking_utils.py", line 17, in log
wandb: ERROR     wandb.log(metrics)
wandb: ERROR   File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/wandb/sdk/lib/preinit.py", line 36, in preinit_wrapper
wandb: ERROR     raise wandb.Error(f"You must call wandb.init() before {name}()")
wandb: ERROR wandb.errors.errors.Error: You must call wandb.init() before wandb.log()
Traceback (most recent call last):
  File "/fast/home/mihir.mahajan/Projects/miles/train.py", line 140, in <module>
    train(args)
  File "/fast/home/mihir.mahajan/Projects/miles/train.py", line 89, in train
    rollout_data_ref = ray.get(rollout_manager.generate.remote(rollout_id))
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py", line 2972, in get
    values, debugger_breakpoint = worker.get_objects(
                                  ^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py", line 1031, in get_objects
    raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(Error): ray::RolloutManager.generate() (pid=721547, ip=10.86.2.80, actor_id=e975bf23916eb598b5f5e00c02000000, repr=<miles.ray.rollout.RolloutManager object at 0x7f2468350620>)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/home/mihir.mahajan/Projects/miles/miles/ray/rollout.py", line 131, in generate
    _log_rollout_data(rollout_id, self.args, data, metrics, time.time() - start_time)
  File "/fast/home/mihir.mahajan/Projects/miles/miles/ray/rollout.py", line 643, in _log_rollout_data
    tracking_utils.log(args, log_dict, step_key="rollout/step")
  File "/fast/home/mihir.mahajan/Projects/miles/miles/utils/tracking_utils.py", line 17, in log
    wandb.log(metrics)
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/wandb/sdk/lib/preinit.py", line 36, in preinit_wrapper
    raise wandb.Error(f"You must call wandb.init() before {name}()")
wandb.errors.errors.Error: You must call wandb.init() before wandb.log()
Traceback (most recent call last):
  File "/fast/home/mihir.mahajan/Projects/miles/train.py", line 140, in <module>
    train(args)
  File "/fast/home/mihir.mahajan/Projects/miles/train.py", line 89, in train
    rollout_data_ref = ray.get(rollout_manager.generate.remote(rollout_id))
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py", line 2972, in get
    values, debugger_breakpoint = worker.get_objects(
                                  ^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py", line 1031, in get_objects
    raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(Error): ray::RolloutManager.generate() (pid=721547, ip=10.86.2.80, actor_id=e975bf23916eb598b5f5e00c02000000, repr=<miles.ray.rollout.RolloutManager object at 0x7f2468350620>)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/home/mihir.mahajan/Projects/miles/miles/ray/rollout.py", line 131, in generate
    _log_rollout_data(rollout_id, self.args, data, metrics, time.time() - start_time)
  File "/fast/home/mihir.mahajan/Projects/miles/miles/ray/rollout.py", line 643, in _log_rollout_data
    tracking_utils.log(args, log_dict, step_key="rollout/step")
  File "/fast/home/mihir.mahajan/Projects/miles/miles/utils/tracking_utils.py", line 17, in log
    wandb.log(metrics)
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/wandb/sdk/lib/preinit.py", line 36, in preinit_wrapper
    raise wandb.Error(f"You must call wandb.init() before {name}()")
wandb.errors.errors.Error: You must call wandb.init() before wandb.log()
(RolloutManager pid=721547) DEBUG: _get_rollout_data returned. Data and metrics obtained.
(RolloutManager pid=721547) DEBUG: _save_debug_rollout_data called.
(RolloutManager pid=721547) DEBUG: Entering finally block of generate method
(RolloutManager pid=721547) DEBUG: Health monitor not started or already stopped. num_new_engines: 0
(RolloutManager pid=721547) DEBUG: generate method finished
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
(RolloutManager pid=721547) [2025-12-11 19:59:20] rollout.py:640 - perf 0: {'perf/rollout_time': 0.7748336791992188, 'perf/longest_sample_tokens_per_sec': 978.2744611506612, 'rollout/response_len/mean': 211.7734375, 'rollout/response_len/median': 151.5, 'rollout/zero_std/count_0': 128, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.0}
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
wandb: 
wandb: 🚀 View run qwen3-0.6b-sft-torch_526x703k-RANK_0 at: https://wandb.ai/instant-uv/crowd-pilot-miles/runs/0cdor63m
wandb: Find logs at: ../../../../fast/home/mihir.mahajan/Projects/miles/wandb/run-20251211_194013-0cdor63m/logs
(FSDPTrainRayActor pid=721548) set_train_parallel_config done
[2025-12-11 19:59:23] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:23] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:23] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:23] selector_events.py:1068 - socket.send() raised exception.
2025-12-11 19:59:29,403	ERR cli.py:73 -- ---------------------------------------
2025-12-11 19:59:29,403	ERR cli.py:74 -- Job 'raysubmit_4DsLVRzyAmBnNWKJ' failed
2025-12-11 19:59:29,403	ERR cli.py:75 -- ---------------------------------------
2025-12-11 19:59:29,403	INFO cli.py:88 -- Status message: Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:20] selector_events.py:1068 - socket.send() raised exception.
wandb: 
wandb: 🚀 View run qwen3-0.6b-sft-torch_526x703k-RANK_0 at: https://wandb.ai/instant-uv/crowd-pilot-miles/runs/0cdor63m
wandb: Find logs at: ../../../../fast/home/mihir.mahajan/Projects/miles/wandb/run-20251211_194013-0cdor63m/logs
(FSDPTrainRayActor pid=721548) set_train_parallel_config done
[2025-12-11 19:59:23] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:23] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:23] selector_events.py:1068 - socket.send() raised exception.
[2025-12-11 19:59:23] selector_events.py:1068 - socket.send() raised exception.

srun: error: hai008: task 0: Exited with exit code 1