[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/run_qwen3-0.6b-sft-torch_36267.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-sft-torch


trap 'echo "Job killed by signal"' USR1

cat $0

source ~/.bashrc

micromamba activate miles

srun sh scripts/run-qwen3-0.6B-torch-sft.shRunning inside SLURM, Job ID 36267.
2025-12-11 12:11:41,707	INFO scripts.py:1371 -- Did not find any active Ray processes.
HAS_NVLINK: 0 (detected 0 NVLink references)
Detected Head Node IP: 10.86.2.80
2025-12-11 12:11:47,864	INFO usage_lib.py:447 -- Usage stats collection is disabled.
2025-12-11 12:11:47,864	INFO scripts.py:919 -- Local node IP: 10.86.2.80
2025-12-11 12:12:04,319	SUCC scripts.py:963 -- --------------------
2025-12-11 12:12:04,319	SUCC scripts.py:964 -- Ray runtime started.
2025-12-11 12:12:04,319	SUCC scripts.py:965 -- --------------------
2025-12-11 12:12:04,320	INFO scripts.py:967 -- Next steps
2025-12-11 12:12:04,320	INFO scripts.py:970 -- To add another node to this Ray cluster, run
2025-12-11 12:12:04,320	INFO scripts.py:973 --   ray start --address='10.86.2.80:6379'
2025-12-11 12:12:04,320	INFO scripts.py:982 -- To connect to this Ray cluster:
2025-12-11 12:12:04,320	INFO scripts.py:984 -- import ray
2025-12-11 12:12:04,320	INFO scripts.py:985 -- ray.init(_node_ip_address='10.86.2.80')
2025-12-11 12:12:04,320	INFO scripts.py:997 -- To submit a Ray job using the Ray Jobs CLI:
2025-12-11 12:12:04,320	INFO scripts.py:998 --   RAY_API_SERVER_ADDRESS='http://10.86.2.80:8265' ray job submit --working-dir . -- python my_script.py
2025-12-11 12:12:04,320	INFO scripts.py:1007 -- See https://docs.ray.io/en/latest/cluster/running-applications/job-submission/index.html 
2025-12-11 12:12:04,320	INFO scripts.py:1011 -- for more information on submitting Ray jobs to the Ray cluster.
2025-12-11 12:12:04,320	INFO scripts.py:1016 -- To terminate the Ray runtime, run
2025-12-11 12:12:04,320	INFO scripts.py:1017 --   ray stop
2025-12-11 12:12:04,320	INFO scripts.py:1020 -- To view the status of the cluster, use
2025-12-11 12:12:04,320	INFO scripts.py:1021 --   ray status
2025-12-11 12:12:04,320	INFO scripts.py:1025 -- To monitor and debug Ray, view the dashboard at 
2025-12-11 12:12:04,320	INFO scripts.py:1026 --   10.86.2.80:8265
2025-12-11 12:12:04,320	INFO scripts.py:1033 -- If connection to the dashboard fails, check your firewall settings and network configuration.
Ray started. Waiting for Dashboard to be ready...
Dashboard is up!
2025-12-11 12:12:13,373	INFO cli.py:41 -- Job submission server address: http://10.86.2.80:8265
2025-12-11 12:12:14,489	SUCC cli.py:65 -- -------------------------------------------------------
2025-12-11 12:12:14,490	SUCC cli.py:66 -- Job 'raysubmit_Zcm6yRB8tdTwDsmb' submitted successfully
2025-12-11 12:12:14,490	SUCC cli.py:67 -- -------------------------------------------------------
2025-12-11 12:12:14,490	INFO cli.py:291 -- Next steps
2025-12-11 12:12:14,490	INFO cli.py:292 -- Query the logs of the job:
2025-12-11 12:12:14,490	INFO cli.py:294 -- ray job logs raysubmit_Zcm6yRB8tdTwDsmb
2025-12-11 12:12:14,490	INFO cli.py:296 -- Query the status of the job:
2025-12-11 12:12:14,490	INFO cli.py:298 -- ray job status raysubmit_Zcm6yRB8tdTwDsmb
2025-12-11 12:12:14,490	INFO cli.py:300 -- Request the job to be stopped:
2025-12-11 12:12:14,490	INFO cli.py:302 -- ray job stop raysubmit_Zcm6yRB8tdTwDsmb
2025-12-11 12:12:14,494	INFO cli.py:312 -- Tailing logs until the job exits (disable with --no-wait):
2025-12-11 12:12:14,206	INFO job_manager.py:568 -- Runtime env is setting up.
Running entrypoint for job raysubmit_Zcm6yRB8tdTwDsmb: python3 train.py --hf-checkpoint /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --ref-load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --rollout-function-path miles.rollout.sft_rollout.generate_rollout --prompt-data /fast/project/HFMI_SynergyUnit/tab_model/huggingface/openhermes2_5.parquet --input-key messages --apply-chat-template --rollout-shuffle --num-epoch 3 --rollout-batch-size 128 --global-batch-size 128 --loss-type sft_loss --calculate-per-token-loss --disable-compute-advantages-and-returns --debug-train-only --optimizer adam --lr 1e-5 --lr-decay-style constant --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.98 --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen3-0.6b-sft-torch --rollout-num-gpus-per-engine 2 --sglang-mem-fraction-static 0.5 --sglang-decode-log-interval 1000 --sglang-chunked-prefill-size 4096 --sglang-attention-backend fa3 --train-backend fsdp --update-weight-buffer-size 536870912 --gradient-checkpointing --attn-implementation flash_attention_3 --train-env-vars {\"PYTORCH_CUDA_ALLOC_CONF\":\"expandable_segments:True\"} --actor-num-gpus-per-node 2 --use-dynamic-batch-size --max-tokens-per-gpu 9216
/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
[2025-12-11 12:12:30] INFO trace.py:52: opentelemetry package is not installed, tracing disabled
2025-12-11 12:12:34,777	INFO worker.py:1696 -- Using address 10.86.2.80:6379 set in the environment variable RAY_ADDRESS
2025-12-11 12:12:34,787	INFO worker.py:1837 -- Connecting to existing Ray cluster at address: 10.86.2.80:6379...
2025-12-11 12:12:34,799	INFO worker.py:2014 -- Connected to Ray cluster. View the dashboard at http://10.86.2.80:8265 
/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/_private/worker.py:2062: FutureWarning: Tip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0
  warnings.warn(
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: WARNING The `shared` mode feature is experimental and may change. Please contact support@wandb.com for guidance and to report any issues.
wandb: setting up run zag2jptr
wandb: Tracking run with wandb version 0.23.1
wandb: Run data is saved locally in /fast/home/mihir.mahajan/Projects/miles/wandb/run-20251211_121239-zag2jptr
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run qwen3-0.6b-sft-torch_9q1lcymp-RANK_0
wandb: ⭐️ View project at https://wandb.ai/instant-uv/crowd-pilot-miles
wandb: 🚀 View run at https://wandb.ai/instant-uv/crowd-pilot-miles/runs/zag2jptr
wandb: Detected [openai] in use.
wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
(pid=429793) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
(pid=429793)   import pynvml  # type: ignore[import]
(RolloutManager pid=429793) 2025-12-11 11:12:53  WARN sgl_model_gateway::mcp::manager: /home/runner/work/sglang/sglang/src/mcp/manager.rs:90: No static MCP servers connected
(RolloutManager pid=429793) [2025-12-11 12:12:53] trace.py:52 - opentelemetry package is not installed, tracing disabled
(RolloutManager pid=429793) [2025-12-11 12:12:53] rollout.py:548 - Launch router with args: RouterArgs(worker_urls=[], host='10.86.2.80', port=4426, mini_lb=False, pd_disaggregation=False, prefill_urls=[], decode_urls=[], policy='cache_aware', prefill_policy=None, decode_policy=None, worker_startup_timeout_secs=1800, worker_startup_check_interval=30, cache_threshold=0.3, balance_abs_threshold=64, balance_rel_threshold=1.5, eviction_interval_secs=120, max_tree_size=67108864, max_payload_size=536870912, bucket_adjust_interval_secs=5, dp_aware=False, enable_igw=False, api_key=None, log_dir=None, log_level='warn', service_discovery=False, selector={}, service_discovery_port=80, service_discovery_namespace=None, prefill_selector={}, decode_selector={}, bootstrap_port_annotation='sglang.ai/bootstrap-port', prometheus_port=5415, prometheus_host='0.0.0.0', request_id_headers=None, request_timeout_secs=14400, max_concurrent_requests=-1, queue_size=100, queue_timeout_secs=60, rate_limit_tokens_per_second=None, cors_allowed_origins=[], retry_max_retries=5, retry_initial_backoff_ms=50, retry_max_backoff_ms=30000, retry_backoff_multiplier=1.5, retry_jitter_factor=0.2, disable_retries=False, health_failure_threshold=3, health_success_threshold=2, health_check_timeout_secs=5, health_check_interval_secs=60, health_check_endpoint='/health', cb_failure_threshold=10, cb_success_threshold=3, cb_timeout_duration_secs=60, cb_window_duration_secs=120, disable_circuit_breaker=False, model_path=None, tokenizer_path=None, chat_template=None, tokenizer_cache_enable_l0=False, tokenizer_cache_l0_max_entries=10000, tokenizer_cache_enable_l1=False, tokenizer_cache_l1_max_memory=52428800, reasoning_parser=None, tool_call_parser=None, mcp_config_path=None, backend='sglang', history_backend='memory', oracle_wallet_path=None, oracle_tns_alias=None, oracle_connect_descriptor=None, oracle_username=None, oracle_password=None, oracle_pool_min=1, oracle_pool_max=16, oracle_pool_timeout_secs=30, postgres_db_url=None, postgres_pool_max=16, client_cert_path=None, client_key_path=None, ca_cert_paths=[], enable_trace=False, otlp_traces_endpoint='localhost:4317')
(RolloutManager pid=429793) [2025-12-11 12:12:56] rollout.py:559 - Router launched at 10.86.2.80:4426
(RolloutManager pid=429793) wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
(RolloutManager pid=429793) wandb: WARNING The `shared` mode feature is experimental and may change. Please contact support@wandb.com for guidance and to report any issues.
(RolloutManager pid=429793) wandb: WARNING Using a boolean value for 'reinit' is deprecated. Use 'return_previous' or 'finish_previous' instead.
(RolloutManager pid=429793) wandb: setting up run zag2jptr
(RolloutManager pid=429793) wandb: Tracking run with wandb version 0.23.1
(RolloutManager pid=429793) wandb: Run data is saved locally in /fast/home/mihir.mahajan/Projects/miles/wandb/run-20251211_121257-zag2jptr
(RolloutManager pid=429793) wandb: Run `wandb offline` to turn off syncing.
(RolloutManager pid=429793) wandb: Resuming run qwen3-0.6b-sft-torch_9q1lcymp-RANK_0
(RolloutManager pid=429793) wandb: ⭐️ View project at https://wandb.ai/instant-uv/crowd-pilot-miles
(RolloutManager pid=429793) wandb: 🚀 View run at https://wandb.ai/instant-uv/crowd-pilot-miles/runs/zag2jptr
(RolloutManager pid=429793) wandb: Detected [openai] in use.
(RolloutManager pid=429793) wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
(RolloutManager pid=429793) wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
(RolloutManager pid=429793) [2025-12-11 12:29:18] rollout.py:61 - import miles.rollout.sft_rollout.generate_rollout as generate_rollout function.
(RolloutManager pid=429793) [2025-12-11 12:29:18] rollout.py:62 - import miles.rollout.sft_rollout.generate_rollout as eval_generate_rollout function.
(pid=439476) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
(pid=439476)   import pynvml  # type: ignore[import]
(pid=439875) /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
(pid=439875)   import pynvml  # type: ignore[import]
(FSDPTrainRayActor pid=439476) [Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
(FSDPTrainRayActor pid=439476) [2025-12-11 12:29:58] train_actor.py:96 - Set NUMA affinity for GPU 0
(FSDPTrainRayActor pid=439476) [2025-12-11 12:29:58] actor.py:217 - [Rank 0] Device mesh (2D): world_size=2, cp_size=1, dp_size=2
(FSDPTrainRayActor pid=439476) [2025-12-11 12:29:58] actor.py:221 - [Rank 0] Mesh shape: (2, 1), dp_rank=0, cp_rank=0
(FSDPTrainRayActor pid=439476) [2025-12-11 12:29:58] actor.py:228 - [Rank 0] Pure DP mode (cp_size=1)
(FSDPTrainRayActor pid=439476) wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
(FSDPTrainRayActor pid=439476) wandb: WARNING The `shared` mode feature is experimental and may change. Please contact support@wandb.com for guidance and to report any issues.
(FSDPTrainRayActor pid=439476) wandb: WARNING Using a boolean value for 'reinit' is deprecated. Use 'return_previous' or 'finish_previous' instead.
(FSDPTrainRayActor pid=439476) wandb: setting up run zag2jptr
(FSDPTrainRayActor pid=439476) wandb: Tracking run with wandb version 0.23.1
(FSDPTrainRayActor pid=439476) wandb: Run data is saved locally in /fast/home/mihir.mahajan/Projects/miles/wandb/run-20251211_122958-zag2jptr
(FSDPTrainRayActor pid=439476) wandb: Run `wandb offline` to turn off syncing.
(FSDPTrainRayActor pid=439476) wandb: Resuming run qwen3-0.6b-sft-torch_9q1lcymp-RANK_0
(FSDPTrainRayActor pid=439476) wandb: ⭐️ View project at https://wandb.ai/instant-uv/crowd-pilot-miles
(FSDPTrainRayActor pid=439476) wandb: 🚀 View run at https://wandb.ai/instant-uv/crowd-pilot-miles/runs/zag2jptr
srun: forcing job termination