[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/run_qwen3-0.6b-sft-torch_36295.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-sft-torch


trap 'echo "Job killed by signal"' USR1

cat $0

source ~/.bashrc

micromamba activate miles

srun sh scripts/run-qwen3-0.6B-torch-sft.shRunning inside SLURM, Job ID 36295.
2025-12-12 15:19:06,219	INFO scripts.py:1371 -- Did not find any active Ray processes.
HAS_NVLINK: 0 (detected 0 NVLink references)
Detected Head Node IP: 10.86.2.20
2025-12-12 15:19:13,158	INFO usage_lib.py:447 -- Usage stats collection is disabled.
2025-12-12 15:19:13,158	INFO scripts.py:919 -- Local node IP: 10.86.2.20
2025-12-12 15:19:34,676	SUCC scripts.py:963 -- --------------------
2025-12-12 15:19:34,676	SUCC scripts.py:964 -- Ray runtime started.
2025-12-12 15:19:34,676	SUCC scripts.py:965 -- --------------------
2025-12-12 15:19:34,676	INFO scripts.py:967 -- Next steps
2025-12-12 15:19:34,676	INFO scripts.py:970 -- To add another node to this Ray cluster, run
2025-12-12 15:19:34,676	INFO scripts.py:973 --   ray start --address='10.86.2.20:6379'
2025-12-12 15:19:34,676	INFO scripts.py:982 -- To connect to this Ray cluster:
2025-12-12 15:19:34,676	INFO scripts.py:984 -- import ray
2025-12-12 15:19:34,676	INFO scripts.py:985 -- ray.init(_node_ip_address='10.86.2.20')
2025-12-12 15:19:34,676	INFO scripts.py:997 -- To submit a Ray job using the Ray Jobs CLI:
2025-12-12 15:19:34,676	INFO scripts.py:998 --   RAY_API_SERVER_ADDRESS='http://10.86.2.20:8265' ray job submit --working-dir . -- python my_script.py
2025-12-12 15:19:34,676	INFO scripts.py:1007 -- See https://docs.ray.io/en/latest/cluster/running-applications/job-submission/index.html 
2025-12-12 15:19:34,676	INFO scripts.py:1011 -- for more information on submitting Ray jobs to the Ray cluster.
2025-12-12 15:19:34,676	INFO scripts.py:1016 -- To terminate the Ray runtime, run
2025-12-12 15:19:34,676	INFO scripts.py:1017 --   ray stop
2025-12-12 15:19:34,676	INFO scripts.py:1020 -- To view the status of the cluster, use
2025-12-12 15:19:34,676	INFO scripts.py:1021 --   ray status
2025-12-12 15:19:34,676	INFO scripts.py:1025 -- To monitor and debug Ray, view the dashboard at 
2025-12-12 15:19:34,676	INFO scripts.py:1026 --   10.86.2.20:8265
2025-12-12 15:19:34,676	INFO scripts.py:1033 -- If connection to the dashboard fails, check your firewall settings and network configuration.
Ray started. Waiting for Dashboard to be ready...
Dashboard is up!
2025-12-12 15:19:43,832	INFO cli.py:41 -- Job submission server address: http://10.86.2.20:8265
2025-12-12 15:19:50,390	SUCC cli.py:65 -- -------------------------------------------------------
2025-12-12 15:19:50,390	SUCC cli.py:66 -- Job 'raysubmit_Zn8JrviLNrk2cGdg' submitted successfully
2025-12-12 15:19:50,390	SUCC cli.py:67 -- -------------------------------------------------------
2025-12-12 15:19:50,390	INFO cli.py:291 -- Next steps
2025-12-12 15:19:50,390	INFO cli.py:292 -- Query the logs of the job:
2025-12-12 15:19:50,390	INFO cli.py:294 -- ray job logs raysubmit_Zn8JrviLNrk2cGdg
2025-12-12 15:19:50,390	INFO cli.py:296 -- Query the status of the job:
2025-12-12 15:19:50,391	INFO cli.py:298 -- ray job status raysubmit_Zn8JrviLNrk2cGdg
2025-12-12 15:19:50,391	INFO cli.py:300 -- Request the job to be stopped:
2025-12-12 15:19:50,391	INFO cli.py:302 -- ray job stop raysubmit_Zn8JrviLNrk2cGdg
2025-12-12 15:19:50,394	INFO cli.py:312 -- Tailing logs until the job exits (disable with --no-wait):
2025-12-12 15:19:50,073	INFO job_manager.py:568 -- Runtime env is setting up.
Running entrypoint for job raysubmit_Zn8JrviLNrk2cGdg: python3 train.py --hf-checkpoint /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --ref-load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --rollout-function-path miles.rollout.sft_rollout.generate_rollout --prompt-data /fast/project/HFMI_SynergyUnit/tab_model/huggingface/openhermes2_5.parquet --input-key messages --apply-chat-template --rollout-shuffle --num-epoch 3 --rollout-batch-size 128 --global-batch-size 128 --loss-type sft_loss --calculate-per-token-loss --disable-compute-advantages-and-returns --debug-train-only --optimizer adam --lr 1e-5 --lr-decay-style constant --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.98 --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen3-0.6b-sft-torch --train-backend fsdp --update-weight-buffer-size 536870912 --gradient-checkpointing --attn-implementation flash_attention_3 --train-env-vars {\"PYTORCH_CUDA_ALLOC_CONF\":\"expandable_segments:True\"} --actor-num-gpus-per-node 2 --use-dynamic-batch-size --max-tokens-per-gpu 9216
/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
[2025-12-12 15:20:09] INFO trace.py:52: opentelemetry package is not installed, tracing disabled
DEBUG: Configured logger.
2025-12-12 15:20:14,157	INFO worker.py:1696 -- Using address 10.86.2.20:6379 set in the environment variable RAY_ADDRESS
2025-12-12 15:20:14,166	INFO worker.py:1837 -- Connecting to existing Ray cluster at address: 10.86.2.20:6379...
2025-12-12 15:20:14,179	INFO worker.py:2014 -- Connected to Ray cluster. View the dashboard at http://10.86.2.20:8265 
/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py:2062: FutureWarning: Tip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0
  warnings.warn(
DEBUG: Created placement groups.
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: WARNING The `shared` mode feature is experimental and may change. Please contact support@wandb.com for guidance and to report any issues.
wandb: setting up run 8zajin0k
wandb: Tracking run with wandb version 0.23.1
wandb: Run data is saved locally in /fast/home/mihir.mahajan/Projects/miles/wandb/run-20251212_152018-8zajin0k
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run qwen3-0.6b-sft-torch_h6463iu3-RANK_0
wandb: ⭐️ View project at https://wandb.ai/instant-uv/crowd-pilot-miles
wandb: 🚀 View run at https://wandb.ai/instant-uv/crowd-pilot-miles/runs/8zajin0k
wandb: Detected [openai] in use.
wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
DEBUG: Initialized tracking.
DEBUG: Created rollout manager.
DEBUG: allocating actor_model
(pid=3049058) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
(pid=3049058)   import pynvml  # type: ignore[import]
(RolloutManager pid=3049058) DEBUG: configure_logger() executed
(RolloutManager pid=3049058) DEBUG: self.args = args executed
(RolloutManager pid=3049058) DEBUG: self.pg = pg executed
(RolloutManager pid=3049058) 2025-12-12 14:20:34  WARN sgl_model_gateway::mcp::manager: /home/runner/work/sglang/sglang/src/mcp/manager.rs:90: No static MCP servers connected
(RolloutManager pid=3049058) [2025-12-12 15:20:34] trace.py:52 - opentelemetry package is not installed, tracing disabled
(RolloutManager pid=3049058) [2025-12-12 15:20:34] rollout.py:586 - Launch router with args: RouterArgs(worker_urls=[], host='10.86.2.20', port=3571, mini_lb=False, pd_disaggregation=False, prefill_urls=[], decode_urls=[], policy='cache_aware', prefill_policy=None, decode_policy=None, worker_startup_timeout_secs=1800, worker_startup_check_interval=30, cache_threshold=0.3, balance_abs_threshold=64, balance_rel_threshold=1.5, eviction_interval_secs=120, max_tree_size=67108864, max_payload_size=536870912, bucket_adjust_interval_secs=5, dp_aware=False, enable_igw=False, api_key=None, log_dir=None, log_level='warn', service_discovery=False, selector={}, service_discovery_port=80, service_discovery_namespace=None, prefill_selector={}, decode_selector={}, bootstrap_port_annotation='sglang.ai/bootstrap-port', prometheus_port=5645, prometheus_host='0.0.0.0', request_id_headers=None, request_timeout_secs=14400, max_concurrent_requests=-1, queue_size=100, queue_timeout_secs=60, rate_limit_tokens_per_second=None, cors_allowed_origins=[], retry_max_retries=5, retry_initial_backoff_ms=50, retry_max_backoff_ms=30000, retry_backoff_multiplier=1.5, retry_jitter_factor=0.2, disable_retries=False, health_failure_threshold=3, health_success_threshold=2, health_check_timeout_secs=5, health_check_interval_secs=60, health_check_endpoint='/health', cb_failure_threshold=10, cb_success_threshold=3, cb_timeout_duration_secs=60, cb_window_duration_secs=120, disable_circuit_breaker=False, model_path=None, tokenizer_path=None, chat_template=None, tokenizer_cache_enable_l0=False, tokenizer_cache_l0_max_entries=10000, tokenizer_cache_enable_l1=False, tokenizer_cache_l1_max_memory=52428800, reasoning_parser=None, tool_call_parser=None, mcp_config_path=None, backend='sglang', history_backend='memory', oracle_wallet_path=None, oracle_tns_alias=None, oracle_connect_descriptor=None, oracle_username=None, oracle_password=None, oracle_pool_min=1, oracle_pool_max=16, oracle_pool_timeout_secs=30, postgres_db_url=None, postgres_pool_max=16, client_cert_path=None, client_key_path=None, ca_cert_paths=[], enable_trace=False, otlp_traces_endpoint='localhost:4317')
(pid=3049059) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
(pid=3049059)   import pynvml  # type: ignore[import]
(RolloutManager pid=3049058) DEBUG: _start_router(args) executed
(RolloutManager pid=3049058) [2025-12-12 15:20:37] rollout.py:597 - Router launched at 10.86.2.20:3571
(RolloutManager pid=3049058) wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
(RolloutManager pid=3049058) wandb: WARNING The `shared` mode feature is experimental and may change. Please contact support@wandb.com for guidance and to report any issues.
(RolloutManager pid=3049058) wandb: WARNING Using a boolean value for 'reinit' is deprecated. Use 'return_previous' or 'finish_previous' instead.
(RolloutManager pid=3049058) wandb: setting up run 8zajin0k
(RolloutManager pid=3049058) wandb: Tracking run with wandb version 0.23.1
(RolloutManager pid=3049058) wandb: Run data is saved locally in /fast/home/mihir.mahajan/Projects/miles/wandb/run-20251212_152037-8zajin0k
(RolloutManager pid=3049058) wandb: Run `wandb offline` to turn off syncing.
(RolloutManager pid=3049058) wandb: Resuming run qwen3-0.6b-sft-torch_h6463iu3-RANK_0
(RolloutManager pid=3049058) wandb: ⭐️ View project at https://wandb.ai/instant-uv/crowd-pilot-miles
(RolloutManager pid=3049058) wandb: 🚀 View run at https://wandb.ai/instant-uv/crowd-pilot-miles/runs/8zajin0k
(RolloutManager pid=3049058) DEBUG: init_tracking executed
(RolloutManager pid=3049058) DEBUG: init_http_client(args) executed
(RolloutManager pid=3049058) wandb: Detected [openai] in use.
(RolloutManager pid=3049058) wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
(RolloutManager pid=3049058) wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
(RolloutManager pid=3049058) DEBUG: data_source_cls loaded
DEBUG: allocated actor_model
DEBUG: critic_model is None
(pid=3049421) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
(pid=3049421)   import pynvml  # type: ignore[import]
(FSDPTrainRayActor pid=3049059) [Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:20:57] train_actor.py:96 - Set NUMA affinity for GPU 0
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:20:57] actor.py:217 - [Rank 0] Device mesh (2D): world_size=2, cp_size=1, dp_size=2
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:20:57] actor.py:221 - [Rank 0] Mesh shape: (2, 1), dp_rank=0, cp_rank=0
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:20:57] actor.py:228 - [Rank 0] Pure DP mode (cp_size=1)
(FSDPTrainRayActor pid=3049059) wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
(FSDPTrainRayActor pid=3049059) wandb: WARNING The `shared` mode feature is experimental and may change. Please contact support@wandb.com for guidance and to report any issues.
(FSDPTrainRayActor pid=3049059) wandb: WARNING Using a boolean value for 'reinit' is deprecated. Use 'return_previous' or 'finish_previous' instead.
(FSDPTrainRayActor pid=3049059) wandb: setting up run 8zajin0k
(FSDPTrainRayActor pid=3049059) wandb: Tracking run with wandb version 0.23.1
(FSDPTrainRayActor pid=3049059) wandb: Run data is saved locally in /fast/home/mihir.mahajan/Projects/miles/wandb/run-20251212_152058-8zajin0k
(FSDPTrainRayActor pid=3049059) wandb: Run `wandb offline` to turn off syncing.
(FSDPTrainRayActor pid=3049059) wandb: Resuming run qwen3-0.6b-sft-torch_h6463iu3-RANK_0
(FSDPTrainRayActor pid=3049059) wandb: ⭐️ View project at https://wandb.ai/instant-uv/crowd-pilot-miles
(FSDPTrainRayActor pid=3049059) wandb: 🚀 View run at https://wandb.ai/instant-uv/crowd-pilot-miles/runs/8zajin0k
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:21:01] actor.py:252 - [Rank 0] tie_word_embeddings=True, loading full model to CPU on all ranks
(FSDPTrainRayActor pid=3049059) Flash Attention 3 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("meta-llama/Llama-3.2-1B", attn_implementation="flash_attention_3", dtype=torch.float16)`
(FSDPTrainRayActor pid=3049059) Flash Attention 3 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("meta-llama/Llama-3.2-1B", attn_implementation="flash_attention_3", dtype=torch.float16)`
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:21:03] actor.py:1081 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
(FSDPTrainRayActor pid=3049421) [2025-12-12 15:20:57] train_actor.py:96 - Set NUMA affinity for GPU 1
(FSDPTrainRayActor pid=3049421) [2025-12-12 15:20:57] actor.py:217 - [Rank 1] Device mesh (2D): world_size=2, cp_size=1, dp_size=2
(FSDPTrainRayActor pid=3049421) [2025-12-12 15:20:57] actor.py:221 - [Rank 1] Mesh shape: (2, 1), dp_rank=1, cp_rank=0
(FSDPTrainRayActor pid=3049421) [2025-12-12 15:20:57] actor.py:228 - [Rank 1] Pure DP mode (cp_size=1)
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049059 [0] NCCL INFO Bootstrap: Using bond0:10.86.2.20<0>
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049059 [0] NCCL INFO cudaDriverVersion 12080
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049059 [0] NCCL INFO NCCL version 2.27.3+cuda12.9
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049059 [0] NCCL INFO Comm config Blocking set to 1
(FSDPTrainRayActor pid=3049421) [Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO NET/Plugin: Could not find: libnccl-net.so. 
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO NCCL_IB_DISABLE set by environment to 1.
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO NET/Socket : Using [0]bond0:10.86.2.20<0>
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Initialized NET plugin Socket
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Assigned NET plugin Socket to comm
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Using network Socket
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO ncclCommInitRankConfig comm 0x55706df04450 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 19000 commId 0x1e5b9eeeb9f1b16f - Init START
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO RAS client listening socket at ::1<28028>
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO Bootstrap timings total 0.001189 (create 0.000025, send 0.000125, recv 0.000302, ring 0.000040, delay 0.000001)
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0.
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO MNNVL busId 0x19000 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO NCCL_P2P_DISABLE set by environment to 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Setting affinity for GPU 0 to 0,24,112,136
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0.
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO comm 0x55706df04450 rank 0 nRanks 2 nNodes 1 localRanks 2 localRank 0 MNNVL 0
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Channel 00/04 : 0 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Channel 01/04 : 0 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Channel 02/04 : 0 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Channel 03/04 : 0 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] -1/-1/-1->0->1 [2] 1/-1/-1->0->-1 [3] -1/-1/-1->0->1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO P2P Chunksize set to 131072
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. 
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Check P2P Type isAllDirectP2p 0 directMode 0
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049776 [0] NCCL INFO [Proxy Service] Device 0 CPU core 112
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049777 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 24
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Channel 00 : 0[0] -> 1[1] via SHM/direct/direct
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Channel 01 : 0[0] -> 1[1] via SHM/direct/direct
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Channel 02 : 0[0] -> 1[1] via SHM/direct/direct
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Channel 03 : 0[0] -> 1[1] via SHM/direct/direct
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Connected all trees
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049780 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 112
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 2 p2p channels per peer
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO CC Off, workFifoBytes 1048576
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so. Using internal tuner plugin.
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO ncclCommInitRankConfig comm 0x55706df04450 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 19000 commId 0x1e5b9eeeb9f1b16f - Init COMPLETE
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 2 total 0.43 (kernels 0.29, alloc 0.02, bootstrap 0.00, allgathers 0.00, topo 0.01, graphs 0.00, connections 0.08, rest 0.02)
(FSDPTrainRayActor pid=3049421) [2025-12-12 15:21:04] lr_scheduler.py:80 - > learning rate decay style: constant
(FSDPTrainRayActor pid=3049421) [2025-12-12 15:21:04] checkpoint.py:97 - [FSDP] No tracker file at /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B/latest_checkpointed_iteration.txt; skipping load.
(FSDPTrainRayActor pid=3049421) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
(FSDPTrainRayActor pid=3049421)   warnings.warn(  # warn only once
DEBUG: got start_rollout_ids
DEBUG: asserted start_rollout_ids length
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:21:04] timer.py:24 - Timer train_wait start
(FSDPTrainRayActor pid=3049059) set_rollout_manager
(FSDPTrainRayActor pid=3049059) set_rollout_manager done
(FSDPTrainRayActor pid=3049059) set_train_parallel_config
(RolloutManager pid=3049058) DEBUG: self.data_source initialized
(RolloutManager pid=3049058) DEBUG: self.generate_rollout loaded
(RolloutManager pid=3049058) DEBUG: self.eval_generate_rollout loaded
(RolloutManager pid=3049058) DEBUG: self.custom_reward_post_process_func initialized to None
(RolloutManager pid=3049058) DEBUG: logger.info for generate_rollout executed
(RolloutManager pid=3049058) DEBUG: logger.info for eval_generate_rollout executed
(RolloutManager pid=3049058) DEBUG: self.all_rollout_engines initialized as empty list (debug_train_only)
(RolloutManager pid=3049058) DEBUG: self.num_new_engines initialized
(RolloutManager pid=3049058) DEBUG: self.nodes_per_engine calculated
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049421 [1] NCCL INFO Bootstrap: Using bond0:10.86.2.20<0>
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049421 [1] NCCL INFO cudaDriverVersion 12080
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049421 [1] NCCL INFO NCCL version 2.27.3+cuda12.9
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049421 [1] NCCL INFO Comm config Blocking set to 1
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO NET/Plugin: Could not find: libnccl-net.so. 
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO NCCL_IB_DISABLE set by environment to 1.
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO NET/Socket : Using [0]bond0:10.86.2.20<0>
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO Initialized NET plugin Socket
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO Assigned NET plugin Socket to comm
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO Using network Socket
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO ncclCommInitRankConfig comm 0x559bc7372fa0 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 3b000 commId 0x1e5b9eeeb9f1b16f - Init START
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO RAS client listening socket at ::1<28028>
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO Bootstrap timings total 0.003841 (create 0.000034, send 0.000123, recv 0.002898, ring 0.000292, delay 0.000001)
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049771 [0] NCCL INFO NCCL_CUMEM_ENABLE set by environment to 0.
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO MNNVL busId 0x3b000 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO NCCL_P2P_DISABLE set by environment to 1
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO Setting affinity for GPU 1 to 0,24,112,136
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO NCCL_NVLS_ENABLE set by environment to 0.
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO comm 0x559bc7372fa0 rank 1 nRanks 2 nNodes 1 localRanks 2 localRank 1 MNNVL 0
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] 0/-1/-1->1->-1 [2] -1/-1/-1->1->0 [3] 0/-1/-1->1->-1
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO P2P Chunksize set to 131072
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. 
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049778 [1] NCCL INFO [Proxy Service] Device 1 CPU core 112
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049779 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 112
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO Channel 03 : 1[1] -> 0[0] via SHM/direct/direct [repeated 4x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO Connected all trees
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049781 [1] NCCL INFO [Proxy Progress] Device 1 CPU core 112
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 2 p2p channels per peer
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so. Using internal tuner plugin.
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO ncclCommInitRankConfig comm 0x559bc7372fa0 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 3b000 commId 0x1e5b9eeeb9f1b16f - Init COMPLETE
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049772 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 2 total 0.43 (kernels 0.29, alloc 0.02, bootstrap 0.00, allgathers 0.00, topo 0.01, graphs 0.00, connections 0.09, rest 0.01)
(FSDPTrainRayActor pid=3049421) set_rollout_manager
(FSDPTrainRayActor pid=3049421) set_rollout_manager done
(RolloutManager pid=3049058) [2025-12-12 15:39:25] rollout.py:73 - import miles.rollout.sft_rollout.generate_rollout as generate_rollout function.
(RolloutManager pid=3049058) [2025-12-12 15:39:25] rollout.py:75 - import miles.rollout.sft_rollout.generate_rollout as eval_generate_rollout function.
(FSDPTrainRayActor pid=3049421) [2025-12-12 15:21:01] actor.py:252 - [Rank 1] tie_word_embeddings=True, loading full model to CPU on all ranks
(FSDPTrainRayActor pid=3049421) Flash Attention 3 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("meta-llama/Llama-3.2-1B", attn_implementation="flash_attention_3", dtype=torch.float16)` [repeated 2x across cluster]
(FSDPTrainRayActor pid=3049421) [2025-12-12 15:21:03] actor.py:1081 - FSDP MixedPrecision Policy: param_dtype=torch.bfloat16, reduce_dtype=torch.float32
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:21:04] lr_scheduler.py:80 - > learning rate decay style: constant
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:21:04] checkpoint.py:97 - [FSDP] No tracker file at /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B/latest_checkpointed_iteration.txt; skipping load.
(FSDPTrainRayActor pid=3049059) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. 
(FSDPTrainRayActor pid=3049059)   warnings.warn(  # warn only once
DEBUG: set rollout_manager for actor_model
DEBUG: loaded rollout_manager data
DEBUG: returning actor_model and critic_model
DEBUG: Updated actor model weights.
DEBUG: Defined offload_train function.
DEBUG: Defined onload_rollout function.
Starting rollout 0.
(RolloutManager pid=3049058) DEBUG: self.rollout_engine_lock initialized
(RolloutManager pid=3049058) DEBUG: self._metric_checker initialized
(RolloutManager pid=3049058) set_train_parallel_config
(RolloutManager pid=3049058) set_train_parallel_config done
(RolloutManager pid=3049058) DEBUG: generate method started
(RolloutManager pid=3049058) DEBUG: Calling _get_rollout_data
(RolloutManager pid=3049058) [2025-12-12 15:39:25] data_source.py:141 - Checkpoint /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B/rollout/global_dataset_state_dict_-1.pt does not exist.
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:25] timer.py:24 - Timer update_weights start
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:25] timer.py:32 - Timer update_weights end (elapsed: 0.0s)
(RolloutManager pid=3049058) [2025-12-12 15:39:25] sft_rollout.py:53 - sft_rollout::generate_rollout example data: sample=Sample(group_index=0, index=0, prompt=[{'content': "How can I sort a list of tuples in ascending order based on the second item in the tuple, the integer value? For example, I have a list like this `[('abc', 121),('abc', 231),('abc', 148), ('abc',221)]` and I want to sort it based on the integer values.", 'role': 'user'}, {'content': "Yes, it is possible to sort a list of tuples based on the second item in the tuples. Python provides a built-in function `sorted()` for this purpose. You can use a lambda function as the key which will be used to sort the list. In the lambda function, select the second item of the tuple (index 1) to sort on that.

Here is the code to do this:

```python
# your list of tuples
tuples_list = [('abc', 121),('abc', 231),('abc', 148), ('abc',221)]

# use sorted function
sorted_tuples_list = sorted(tuples_list, key = lambda x: x[1])

# print the sorted list
print(sorted_tuples_list)
```

In this code, `sorted()` is a built-in Python function that takes an iterable and returns a new sorted list of the elements in the iterable. The `key` parameter is a function that takes an element from the iterable and returns a value that will be used for sorting. In this case, we're using a lambda function that takes a tuple and returns the second element of the tuple (i.e., `x[1]`), so the list will be sorted based on the second elements of the tuples.

When you run this code, it will print:

```python
[('abc', 121), ('abc', 148), ('abc', 221), ('abc', 231)]
```

This is your original list, but sorted in ascending order based on the second item of the tuples.", 'role': 'assistant'}], tokens=[151644, 872, 198, 4340, 646, 358, 3378, 264, 1140, 315, 45225, 304, 35388, 1973, 3118, 389, 279, 2086, 1509, 304, 279, 14405, 11, 279, 7546, 897, 30, 1752, 3110, 11, 358, 614, 264, 1140, 1075, 419, 77644, 492, 13683, 516, 220, 16, 17, 16, 701, 492, 13683, 516, 220, 17, 18, 16, 701, 492, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 17, 17, 16, 7252, 63, 323, 358, 1366, 311, 3378, 432, 3118, 389, 279, 7546, 2750, 13, 151645, 198, 151644, 77091, 198, 9454, 11, 432, 374, 3204, 311, 3378, 264, 1140, 315, 45225, 3118, 389, 279, 2086, 1509, 304, 279, 45225, 13, 13027, 5707, 264, 5798, 3419, 729, 1565, 28298, 54258, 369, 419, 7428, 13, 1446, 646, 990, 264, 12459, 729, 438, 279, 1376, 892, 686, 387, 1483, 311, 3378, 279, 1140, 13, 758, 279, 12459, 729, 11, 3293, 279, 2086, 1509, 315, 279, 14405, 320, 1252, 220, 16, 8, 311, 3378, 389, 429, 382, 8420, 374, 279, 2038, 311, 653, 419, 1447, 73594, 12669, 198, 2, 697, 1140, 315, 45225, 198, 83, 28323, 2019, 284, 51128, 13683, 516, 220, 16, 17, 16, 701, 492, 13683, 516, 220, 17, 18, 16, 701, 492, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 17, 17, 16, 27771, 2, 990, 10615, 729, 198, 28298, 89269, 2019, 284, 10615, 1155, 28323, 2019, 11, 1376, 284, 12459, 856, 25, 856, 58, 16, 9604, 2, 1173, 279, 10615, 1140, 198, 1350, 44386, 89269, 2019, 340, 13874, 19324, 641, 419, 2038, 11, 1565, 28298, 54258, 374, 264, 5798, 3419, 13027, 729, 429, 4990, 458, 50834, 323, 4675, 264, 501, 10615, 1140, 315, 279, 5424, 304, 279, 50834, 13, 576, 1565, 792, 63, 5733, 374, 264, 729, 429, 4990, 458, 2392, 504, 279, 50834, 323, 4675, 264, 897, 429, 686, 387, 1483, 369, 28273, 13, 758, 419, 1142, 11, 582, 2299, 1667, 264, 12459, 729, 429, 4990, 264, 14405, 323, 4675, 279, 2086, 2392, 315, 279, 14405, 320, 72, 1734, 2572, 1565, 87, 58, 16, 60, 63, 701, 773, 279, 1140, 686, 387, 10615, 3118, 389, 279, 2086, 5424, 315, 279, 45225, 382, 4498, 498, 1598, 419, 2038, 11, 432, 686, 1173, 1447, 73594, 12669, 198, 58, 492, 13683, 516, 220, 16, 17, 16, 701, 4319, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 220, 17, 17, 16, 701, 4319, 13683, 516, 220, 17, 18, 16, 5563, 13874, 19324, 1986, 374, 697, 4024, 1140, 11, 714, 10615, 304, 35388, 1973, 3118, 389, 279, 2086, 1509, 315, 279, 45225, 13, 151645, 198], multimodal_inputs=None, response='', response_length=335, label=None, reward=0, loss_mask=[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], weight_versions=[], rollout_log_probs=None, rollout_routed_experts=None, remove_sample=False, status=<Status.PENDING: 'pending'>, metadata={}, train_metadata=None, spec_info=<miles.utils.types.Sample.SpecInfo object at 0x7efd7bad5940>) (raw)messages=[{'content': "How can I sort a list of tuples in ascending order based on the second item in the tuple, the integer value? For example, I have a list like this `[('abc', 121),('abc', 231),('abc', 148), ('abc',221)]` and I want to sort it based on the integer values.", 'role': 'user'}, {'content': "Yes, it is possible to sort a list of tuples based on the second item in the tuples. Python provides a built-in function `sorted()` for this purpose. You can use a lambda function as the key which will be used to sort the list. In the lambda function, select the second item of the tuple (index 1) to sort on that.

Here is the code to do this:

```python
# your list of tuples
tuples_list = [('abc', 121),('abc', 231),('abc', 148), ('abc',221)]

# use sorted function
sorted_tuples_list = sorted(tuples_list, key = lambda x: x[1])

# print the sorted list
print(sorted_tuples_list)
```

In this code, `sorted()` is a built-in Python function that takes an iterable and returns a new sorted list of the elements in the iterable. The `key` parameter is a function that takes an element from the iterable and returns a value that will be used for sorting. In this case, we're using a lambda function that takes a tuple and returns the second element of the tuple (i.e., `x[1]`), so the list will be sorted based on the second elements of the tuples.

When you run this code, it will print:

```python
[('abc', 121), ('abc', 148), ('abc', 221), ('abc', 231)]
```

This is your original list, but sorted in ascending order based on the second item of the tuples.", 'role': 'assistant'}] (raw)token_ids=[151644, 872, 198, 4340, 646, 358, 3378, 264, 1140, 315, 45225, 304, 35388, 1973, 3118, 389, 279, 2086, 1509, 304, 279, 14405, 11, 279, 7546, 897, 30, 1752, 3110, 11, 358, 614, 264, 1140, 1075, 419, 77644, 492, 13683, 516, 220, 16, 17, 16, 701, 492, 13683, 516, 220, 17, 18, 16, 701, 492, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 17, 17, 16, 7252, 63, 323, 358, 1366, 311, 3378, 432, 3118, 389, 279, 7546, 2750, 13, 151645, 198, 151644, 77091, 198, 9454, 11, 432, 374, 3204, 311, 3378, 264, 1140, 315, 45225, 3118, 389, 279, 2086, 1509, 304, 279, 45225, 13, 13027, 5707, 264, 5798, 3419, 729, 1565, 28298, 54258, 369, 419, 7428, 13, 1446, 646, 990, 264, 12459, 729, 438, 279, 1376, 892, 686, 387, 1483, 311, 3378, 279, 1140, 13, 758, 279, 12459, 729, 11, 3293, 279, 2086, 1509, 315, 279, 14405, 320, 1252, 220, 16, 8, 311, 3378, 389, 429, 382, 8420, 374, 279, 2038, 311, 653, 419, 1447, 73594, 12669, 198, 2, 697, 1140, 315, 45225, 198, 83, 28323, 2019, 284, 51128, 13683, 516, 220, 16, 17, 16, 701, 492, 13683, 516, 220, 17, 18, 16, 701, 492, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 17, 17, 16, 27771, 2, 990, 10615, 729, 198, 28298, 89269, 2019, 284, 10615, 1155, 28323, 2019, 11, 1376, 284, 12459, 856, 25, 856, 58, 16, 9604, 2, 1173, 279, 10615, 1140, 198, 1350, 44386, 89269, 2019, 340, 13874, 19324, 641, 419, 2038, 11, 1565, 28298, 54258, 374, 264, 5798, 3419, 13027, 729, 429, 4990, 458, 50834, 323, 4675, 264, 501, 10615, 1140, 315, 279, 5424, 304, 279, 50834, 13, 576, 1565, 792, 63, 5733, 374, 264, 729, 429, 4990, 458, 2392, 504, 279, 50834, 323, 4675, 264, 897, 429, 686, 387, 1483, 369, 28273, 13, 758, 419, 1142, 11, 582, 2299, 1667, 264, 12459, 729, 429, 4990, 264, 14405, 323, 4675, 279, 2086, 2392, 315, 279, 14405, 320, 72, 1734, 2572, 1565, 87, 58, 16, 60, 63, 701, 773, 279, 1140, 686, 387, 10615, 3118, 389, 279, 2086, 5424, 315, 279, 45225, 382, 4498, 498, 1598, 419, 2038, 11, 432, 686, 1173, 1447, 73594, 12669, 198, 58, 492, 13683, 516, 220, 16, 17, 16, 701, 4319, 13683, 516, 220, 16, 19, 23, 701, 4319, 13683, 516, 220, 17, 17, 16, 701, 4319, 13683, 516, 220, 17, 18, 16, 5563, 13874, 19324, 1986, 374, 697, 4024, 1140, 11, 714, 10615, 304, 35388, 1973, 3118, 389, 279, 2086, 1509, 315, 279, 45225, 13, 151645, 198] (raw)loss_mask=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] response_length=335
Generated rollout data for rollout 0.
(RolloutManager pid=3049058) DEBUG: _get_rollout_data returned. Data and metrics obtained.
(RolloutManager pid=3049058) DEBUG: _save_debug_rollout_data called.
(RolloutManager pid=3049058) DEBUG: _log_rollout_data called.
(RolloutManager pid=3049058) DEBUG: _convert_samples_to_train_data called. Data converted.
(RolloutManager pid=3049058) DEBUG: Entering finally block of generate method
(RolloutManager pid=3049058) DEBUG: Health monitor not started or already stopped. num_new_engines: 0
(RolloutManager pid=3049058) DEBUG: generate method finished
(RolloutManager pid=3049058) [2025-12-12 15:39:25] rollout.py:639 - perf 0: {'perf/rollout_time': 0.771848201751709, 'perf/longest_sample_tokens_per_sec': 982.0583869726191, 'rollout/response_len/mean': 211.7734375, 'rollout/response_len/median': 151.5, 'rollout/zero_std/count_0': 128, 'rollout/repetition_frac': 0.0, 'rollout/truncated_ratio': 0.0}
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:25] timer.py:32 - Timer train_wait end (elapsed: 1100.9s)
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:25] timer.py:24 - Timer train start
(FSDPTrainRayActor pid=3049059) hai002:3049059:3049059 [0] NCCL INFO Comm config Blocking set to 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Assigned NET plugin Socket to comm
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Using network Socket
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO ncclCommInitRankConfig comm 0x55706e940840 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 19000 commId 0x3d39ad693d3f9a61 - Init START
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Bootstrap timings total 0.000926 (create 0.000023, send 0.000092, recv 0.000610, ring 0.000020, delay 0.000000)
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO MNNVL busId 0x19000 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Setting affinity for GPU 0 to 0,24,112,136
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO comm 0x55706e940840 rank 0 nRanks 2 nNodes 1 localRanks 2 localRank 0 MNNVL 0
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Channel 00/04 : 0 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Channel 01/04 : 0 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Channel 02/04 : 0 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Channel 03/04 : 0 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] -1/-1/-1->0->1 [2] 1/-1/-1->0->-1 [3] -1/-1/-1->0->1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO P2P Chunksize set to 131072
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Check P2P Type isAllDirectP2p 0 directMode 0
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060045 [0] NCCL INFO [Proxy Service] Device 0 CPU core 0
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060047 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 0
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Connected all trees
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060049 [0] NCCL INFO [Proxy Progress] Device 0 CPU core 24
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 2 p2p channels per peer
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO CC Off, workFifoBytes 1048576
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO ncclCommInitRankConfig comm 0x55706e940840 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 19000 commId 0x3d39ad693d3f9a61 - Init COMPLETE
(FSDPTrainRayActor pid=3049059) hai002:3049059:3060043 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 2 total 0.07 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.01, graphs 0.00, connections 0.05, rest 0.00)
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:25] timer.py:24 - Timer log_probs start
(FSDPTrainRayActor pid=3049059) 
log_probs:   0%|          | 0/3 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
(FSDPTrainRayActor pid=3049421) `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
(FSDPTrainRayActor pid=3049059) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/utils/checkpoint.py:85: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
(FSDPTrainRayActor pid=3049059)   warnings.warn(
(FSDPTrainRayActor pid=3049059) 
log_probs:  33%|███▎      | 1/3 [00:01<00:03,  1.98s/it]/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/utils/checkpoint.py:85: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
(FSDPTrainRayActor pid=3049421) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/utils/checkpoint.py:85: UserWarning: None of the inputs have requires_grad=True. Gradients will be None [repeated 2x across cluster]
(FSDPTrainRayActor pid=3049421)   warnings.warn( [repeated 3x across cluster]
(FSDPTrainRayActor pid=3049059) 
log_probs:  67%|██████▋   | 2/3 [00:07<00:04,  4.22s/it]
(FSDPTrainRayActor pid=3049059) 
log_probs: 100%|██████████| 3/3 [00:07<00:00,  2.35s/it]
log_probs: 100%|██████████| 3/3 [00:07<00:00,  2.63s/it]
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:33] timer.py:32 - Timer log_probs end (elapsed: 7.9s)
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:33] actor.py:521 - rollout 0: {'rollout/raw_reward': 0.0, 'rollout/log_probs': -1.9445464611053467, 'rollout/rollout_log_probs': 0.0, 'rollout/advantages': 0.0, 'rollout/returns': 0.0}
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:33] timer.py:24 - Timer actor_train start
(FSDPTrainRayActor pid=3049059) 
actor_train:   0%|          | 0/3 [00:00<?, ?it/s]
Traceback (most recent call last):
  File "/fast/home/mihir.mahajan/Projects/miles/train.py", line 140, in <module>
    train(args)
  File "/fast/home/mihir.mahajan/Projects/miles/train.py", line 105, in train
    ray.get(actor_model.async_train(rollout_id, rollout_data_ref))
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py", line 2972, in get
    values, debugger_breakpoint = worker.get_objects(
                                  ^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py", line 1031, in get_objects
    raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(AssertionError): ray::FSDPTrainRayActor.train() (pid=3049059, ip=10.86.2.20, actor_id=db8348f049b98c4d4dc9ac2f02000000, repr=<miles.backends.fsdp_utils.actor.FSDPTrainRayActor object at 0x7f0a491e5760>)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/home/mihir.mahajan/Projects/miles/miles/backends/fsdp_utils/actor.py", line 485, in train
    self._train_core(rollout_id=rollout_id, rollout_data=rollout_data)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/home/mihir.mahajan/Projects/miles/miles/backends/fsdp_utils/actor.py", line 559, in _train_core
    self._train_step(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/home/mihir.mahajan/Projects/miles/miles/backends/fsdp_utils/actor.py", line 674, in _train_step
    assert not self.args.calculate_per_token_loss, "calculate_per_token_loss not yet implemented"
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: calculate_per_token_loss not yet implemented
Traceback (most recent call last):
  File "/fast/home/mihir.mahajan/Projects/miles/train.py", line 140, in <module>
    train(args)
  File "/fast/home/mihir.mahajan/Projects/miles/train.py", line 105, in train
    ray.get(actor_model.async_train(rollout_id, rollout_data_ref))
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py", line 2972, in get
    values, debugger_breakpoint = worker.get_objects(
                                  ^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py", line 1031, in get_objects
    raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(AssertionError): ray::FSDPTrainRayActor.train() (pid=3049059, ip=10.86.2.20, actor_id=db8348f049b98c4d4dc9ac2f02000000, repr=<miles.backends.fsdp_utils.actor.FSDPTrainRayActor object at 0x7f0a491e5760>)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/home/mihir.mahajan/Projects/miles/miles/backends/fsdp_utils/actor.py", line 485, in train
    self._train_core(rollout_id=rollout_id, rollout_data=rollout_data)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/home/mihir.mahajan/Projects/miles/miles/backends/fsdp_utils/actor.py", line 559, in _train_core
    self._train_step(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/home/mihir.mahajan/Projects/miles/miles/backends/fsdp_utils/actor.py", line 674, in _train_step
    assert not self.args.calculate_per_token_loss, "calculate_per_token_loss not yet implemented"
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: calculate_per_token_loss not yet implemented
(FSDPTrainRayActor pid=3049059) 
actor_train:   0%|          | 0/3 [00:02<?, ?it/s]
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:36] timer.py:32 - Timer actor_train end (elapsed: 2.4s)
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:36] timer.py:32 - Timer train end (elapsed: 10.5s)
(FSDPTrainRayActor pid=3049059) [2025-12-12 15:39:36] timer.py:24 - Timer train_wait start
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:36] selector_events.py:1068 - socket.send() raised exception.
wandb: 
wandb: 🚀 View run qwen3-0.6b-sft-torch_h6463iu3-RANK_0 at: https://wandb.ai/instant-uv/crowd-pilot-miles/runs/8zajin0k
wandb: Find logs at: ../../../../fast/home/mihir.mahajan/Projects/miles/wandb/run-20251212_152018-8zajin0k/logs
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO Channel 03 : 1[1] -> 0[0] via SHM/direct/direct [repeated 8x across cluster]
(FSDPTrainRayActor pid=3049059) set_train_parallel_config done
(FSDPTrainRayActor pid=3049421) hai002:3049421:3049421 [1] NCCL INFO Comm config Blocking set to 1
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO Assigned NET plugin Socket to comm
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO Using network Socket
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO ncclCommInitRankConfig comm 0x559bc259d510 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 3b000 commId 0x3d39ad693d3f9a61 - Init START
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO Bootstrap timings total 0.000392 (create 0.000020, send 0.000109, recv 0.000137, ring 0.000023, delay 0.000000)
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO MNNVL busId 0x3b000 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO Setting affinity for GPU 1 to 0,24,112,136
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO comm 0x559bc259d510 rank 1 nRanks 2 nNodes 1 localRanks 2 localRank 1 MNNVL 0
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] 0/-1/-1->1->-1 [2] -1/-1/-1->1->0 [3] 0/-1/-1->1->-1
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO P2P Chunksize set to 131072
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060046 [1] NCCL INFO [Proxy Service] Device 1 CPU core 24
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060048 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 24
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO Connected all trees
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060050 [1] NCCL INFO [Proxy Progress] Device 1 CPU core 0
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO 4 coll channels, 4 collnet channels, 0 nvls channels, 4 p2p channels, 2 p2p channels per peer
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO ncclCommInitRankConfig comm 0x559bc259d510 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 3b000 commId 0x3d39ad693d3f9a61 - Init COMPLETE
(FSDPTrainRayActor pid=3049421) hai002:3049421:3060044 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 2 total 0.07 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.01, graphs 0.00, connections 0.05, rest 0.00)
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
2025-12-12 15:39:47,050	ERR cli.py:73 -- ---------------------------------------
2025-12-12 15:39:47,050	ERR cli.py:74 -- Job 'raysubmit_Zn8JrviLNrk2cGdg' failed
2025-12-12 15:39:47,050	ERR cli.py:75 -- ---------------------------------------
2025-12-12 15:39:47,050	INFO cli.py:88 -- Status message: Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.
[2025-12-12 15:39:40] selector_events.py:1068 - socket.send() raised exception.

srun: error: hai002: task 0: Exited with exit code 1