[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch-crowd-code/run_qwen3-0.6b-sft-torch-crowd-code_36316.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch-crowd-code/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch-crowd-code/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-sft-torch-crowd-code


trap 'echo "Job killed by signal"' USR1

cat $0

source ~/.bashrc

micromamba activate miles

srun sh scripts/run-qwen3-0.6B-torch-sft-crowd-code.shRunning inside SLURM, Job ID 36316.
2025-12-13 15:20:27,081	INFO scripts.py:1371 -- Did not find any active Ray processes.
HAS_NVLINK: 0 (detected 0 NVLink references)
Detected Head Node IP: 10.86.2.40
2025-12-13 15:20:33,256	INFO usage_lib.py:447 -- Usage stats collection is disabled.
2025-12-13 15:20:33,257	INFO scripts.py:919 -- Local node IP: 10.86.2.40
2025-12-13 15:20:45,921	SUCC scripts.py:963 -- --------------------
2025-12-13 15:20:45,921	SUCC scripts.py:964 -- Ray runtime started.
2025-12-13 15:20:45,921	SUCC scripts.py:965 -- --------------------
2025-12-13 15:20:45,922	INFO scripts.py:967 -- Next steps
2025-12-13 15:20:45,922	INFO scripts.py:970 -- To add another node to this Ray cluster, run
2025-12-13 15:20:45,922	INFO scripts.py:973 --   ray start --address='10.86.2.40:6379'
2025-12-13 15:20:45,922	INFO scripts.py:982 -- To connect to this Ray cluster:
2025-12-13 15:20:45,922	INFO scripts.py:984 -- import ray
2025-12-13 15:20:45,922	INFO scripts.py:985 -- ray.init(_node_ip_address='10.86.2.40')
2025-12-13 15:20:45,922	INFO scripts.py:997 -- To submit a Ray job using the Ray Jobs CLI:
2025-12-13 15:20:45,922	INFO scripts.py:998 --   RAY_API_SERVER_ADDRESS='http://10.86.2.40:8265' ray job submit --working-dir . -- python my_script.py
2025-12-13 15:20:45,922	INFO scripts.py:1007 -- See https://docs.ray.io/en/latest/cluster/running-applications/job-submission/index.html 
2025-12-13 15:20:45,922	INFO scripts.py:1011 -- for more information on submitting Ray jobs to the Ray cluster.
2025-12-13 15:20:45,922	INFO scripts.py:1016 -- To terminate the Ray runtime, run
2025-12-13 15:20:45,922	INFO scripts.py:1017 --   ray stop
2025-12-13 15:20:45,922	INFO scripts.py:1020 -- To view the status of the cluster, use
2025-12-13 15:20:45,922	INFO scripts.py:1021 --   ray status
2025-12-13 15:20:45,922	INFO scripts.py:1025 -- To monitor and debug Ray, view the dashboard at 
2025-12-13 15:20:45,922	INFO scripts.py:1026 --   10.86.2.40:8265
2025-12-13 15:20:45,922	INFO scripts.py:1033 -- If connection to the dashboard fails, check your firewall settings and network configuration.
Ray started. Waiting for Dashboard to be ready...
Dashboard is up!
2025-12-13 15:21:00,114	INFO cli.py:41 -- Job submission server address: http://10.86.2.40:8265
2025-12-13 15:21:06,564	SUCC cli.py:65 -- -------------------------------------------------------
2025-12-13 15:21:06,564	SUCC cli.py:66 -- Job 'raysubmit_MST4XU4FXkhk8Csq' submitted successfully
2025-12-13 15:21:06,564	SUCC cli.py:67 -- -------------------------------------------------------
2025-12-13 15:21:06,564	INFO cli.py:291 -- Next steps
2025-12-13 15:21:06,564	INFO cli.py:292 -- Query the logs of the job:
2025-12-13 15:21:06,564	INFO cli.py:294 -- ray job logs raysubmit_MST4XU4FXkhk8Csq
2025-12-13 15:21:06,564	INFO cli.py:296 -- Query the status of the job:
2025-12-13 15:21:06,564	INFO cli.py:298 -- ray job status raysubmit_MST4XU4FXkhk8Csq
2025-12-13 15:21:06,564	INFO cli.py:300 -- Request the job to be stopped:
2025-12-13 15:21:06,564	INFO cli.py:302 -- ray job stop raysubmit_MST4XU4FXkhk8Csq
2025-12-13 15:21:06,567	INFO cli.py:312 -- Tailing logs until the job exits (disable with --no-wait):
2025-12-13 15:21:06,270	INFO job_manager.py:568 -- Runtime env is setting up.
Running entrypoint for job raysubmit_MST4XU4FXkhk8Csq: python3 train.py --hf-checkpoint /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --ref-load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B --rollout-function-path miles.rollout.sft_rollout.generate_rollout --prompt-data /fast/project/HFMI_SynergyUnit/tab_model/huggingface/nemo_hf_part_jsonl_4k_tokens.parquet --input-key messages --apply-chat-template --rollout-shuffle --num-epoch 3 --rollout-batch-size 16 --global-batch-size 16 --loss-type sft_loss --calculate-per-token-loss --disable-compute-advantages-and-returns --num-rollout 2000 --debug-train-only --optimizer adam --lr 1e-4 --lr-decay-style WSD --lr-wsd-decay-style linear --lr-warmup-iters 100 --lr_decay_iters 1900 --lr-wsd-decay-iters 500 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.98 --use-wandb --wandb-project crowd-pilot-miles --wandb-team instant-uv --wandb-group qwen3-0.6b-sft-torch --train-backend fsdp --update-weight-buffer-size 536870912 --gradient-checkpointing --attn-implementation flash_attention_3 --train-env-vars {\"PYTORCH_CUDA_ALLOC_CONF\":\"expandable_segments:True\"} --actor-num-gpus-per-node 2 --use-dynamic-batch-size --max-tokens-per-gpu 9216 --actor-num-nodes 1 --actor-num-gpus-per-node 2 --colocate --rollout-max-context-len 8192 --rollout-max-prompt-len 8000 --rollout-max-response-len 8192 --use-fault-tolerance --dump-details /fast/project/HFMI_SynergyUnit/tab_model/huggingface/shared_data/qwen3-600M-fsdp-1116-noref/dump_details
/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
[2025-12-13 15:21:23] INFO trace.py:52: opentelemetry package is not installed, tracing disabled
usage: FSDP SFT Training (miles) [-h] [--config CONFIG]
                                 [--optimizer OPTIMIZER] [--lr LR]
                                 [--lr-warmup-init LR_WARMUP_INIT]
                                 [--min-lr MIN_LR]
                                 [--lr-decay-style LR_DECAY_STYLE]
                                 [--lr-decay-iters LR_DECAY_ITERS]
                                 [--lr-warmup-iters LR_WARMUP_ITERS]
                                 [--lr-warmup-fraction LR_WARMUP_FRACTION]
                                 [--lr-wsd-decay-iters LR_WSD_DECAY_ITERS]
                                 [--lr-wsd-decay-style LR_WSD_DECAY_STYLE]
                                 [--use-checkpoint-lr-scheduler]
                                 [--override-lr-scheduler]
                                 [--weight-decay WEIGHT_DECAY]
                                 [--adam-beta1 ADAM_BETA1]
                                 [--adam-beta2 ADAM_BETA2]
                                 [--adam-eps ADAM_EPS]
                                 [--warmup-ratio WARMUP_RATIO]
                                 [--attn-implementation ATTN_IMPLEMENTATION]
                                 [--wandb-project WANDB_PROJECT]
                                 [--wandb-run-name WANDB_RUN_NAME]
                                 [--gradient-checkpointing] [--fp16]
                                 [--fsdp-state-dict-cpu-offload]
                                 [--fsdp-cpu-offload]
                                 [--fsdp-cpu-backend FSDP_CPU_BACKEND]
                                 [--deterministic-mode]
                                 [--context-parallel-size CONTEXT_PARALLEL_SIZE]
                                 [--record-memory-history]
                                 [--memory-snapshot-path MEMORY_SNAPSHOT_PATH]
                                 [--use-pytorch-profiler]
                                 [--profile-step-start PROFILE_STEP_START]
                                 [--profile-step-end PROFILE_STEP_END]
                                 [--tensorboard-dir TENSORBOARD_DIR]
                                 [--actor-num-nodes ACTOR_NUM_NODES]
                                 [--actor-num-gpus-per-node ACTOR_NUM_GPUS_PER_NODE]
                                 [--critic-num-nodes CRITIC_NUM_NODES]
                                 [--critic-num-gpus-per-node CRITIC_NUM_GPUS_PER_NODE]
                                 [--rollout-num-gpus ROLLOUT_NUM_GPUS]
                                 [--rollout-num-gpus-per-engine ROLLOUT_NUM_GPUS_PER_ENGINE]
                                 [--num-gpus-per-node NUM_GPUS_PER_NODE]
                                 [--colocate] [--offload]
                                 [--offload-train | --no-offload-train]
                                 [--offload-rollout | --no-offload-rollout]
                                 [--distributed-backend DISTRIBUTED_BACKEND]
                                 [--distributed-timeout-minutes DISTRIBUTED_TIMEOUT_MINUTES]
                                 [--train-backend {megatron,fsdp}]
                                 [--true-on-policy-mode]
                                 [--train-env-vars TRAIN_ENV_VARS]
                                 [--train-memory-margin-bytes TRAIN_MEMORY_MARGIN_BYTES]
                                 [--disable-weights-backuper]
                                 [--megatron-to-hf-mode {raw,bridge}]
                                 [--hf-checkpoint HF_CHECKPOINT]
                                 [--use-hf-config-for-megatron]
                                 [--model-name MODEL_NAME]
                                 [--rollout-function-path ROLLOUT_FUNCTION_PATH]
                                 [--rollout-temperature ROLLOUT_TEMPERATURE]
                                 [--rollout-top-p ROLLOUT_TOP_P]
                                 [--rollout-top-k ROLLOUT_TOP_K]
                                 [--rollout-max-context-len ROLLOUT_MAX_CONTEXT_LEN]
                                 [--rollout-max-prompt-len ROLLOUT_MAX_PROMPT_LEN]
                                 [--rollout-max-response-len ROLLOUT_MAX_RESPONSE_LEN]
                                 [--rollout-skip-special-tokens]
                                 [--rollout-stop ROLLOUT_STOP [ROLLOUT_STOP ...]]
                                 [--rollout-stop-token-ids ROLLOUT_STOP_TOKEN_IDS [ROLLOUT_STOP_TOKEN_IDS ...]]
                                 [--rollout-shuffle]
                                 [--rollout-seed ROLLOUT_SEED]
                                 [--over-sampling-batch-size OVER_SAMPLING_BATCH_SIZE]
                                 [--dynamic-sampling-filter-path DYNAMIC_SAMPLING_FILTER_PATH]
                                 [--partial-rollout]
                                 [--custom-generate-function-path CUSTOM_GENERATE_FUNCTION_PATH]
                                 [--buffer-filter-path BUFFER_FILTER_PATH]
                                 [--update-weight-buffer-size UPDATE_WEIGHT_BUFFER_SIZE]
                                 [--update-weights-interval UPDATE_WEIGHTS_INTERVAL]
                                 [--keep-old-actor]
                                 [--rollout-data-postprocess-path ROLLOUT_DATA_POSTPROCESS_PATH]
                                 [--rollout-external]
                                 [--rollout-external-engine-addrs ROLLOUT_EXTERNAL_ENGINE_ADDRS [ROLLOUT_EXTERNAL_ENGINE_ADDRS ...]]
                                 [--use-fault-tolerance]
                                 [--rollout-health-check-interval ROLLOUT_HEALTH_CHECK_INTERVAL]
                                 [--rollout-health-check-timeout ROLLOUT_HEALTH_CHECK_TIMEOUT]
                                 [--rollout-health-check-first-wait ROLLOUT_HEALTH_CHECK_FIRST_WAIT]
                                 [--num-rollout NUM_ROLLOUT]
                                 [--num-epoch NUM_EPOCH]
                                 [--disable-rollout-global-dataset]
                                 [--data-source-path DATA_SOURCE_PATH]
                                 [--prompt-data PROMPT_DATA]
                                 [--apply-chat-template]
                                 [--apply-chat-template-kwargs APPLY_CHAT_TEMPLATE_KWARGS]
                                 [--input-key INPUT_KEY]
                                 [--label-key LABEL_KEY]
                                 [--multimodal-keys MULTIMODAL_KEYS]
                                 [--metadata-key METADATA_KEY]
                                 [--tool-key TOOL_KEY]
                                 [--start-rollout-id START_ROLLOUT_ID]
                                 --rollout-batch-size ROLLOUT_BATCH_SIZE
                                 [--n-samples-per-prompt N_SAMPLES_PER_PROMPT]
                                 [--global-batch-size GLOBAL_BATCH_SIZE]
                                 [--num-steps-per-rollout NUM_STEPS_PER_ROLLOUT]
                                 [--micro-batch-size MICRO_BATCH_SIZE]
                                 [--balance-data] [--use-dynamic-batch-size]
                                 [--max-tokens-per-gpu MAX_TOKENS_PER_GPU]
                                 [--log-probs-max-tokens-per-gpu LOG_PROBS_MAX_TOKENS_PER_GPU]
                                 [--eval-function-path EVAL_FUNCTION_PATH]
                                 [--eval-interval EVAL_INTERVAL]
                                 [--eval-prompt-data EVAL_PROMPT_DATA [EVAL_PROMPT_DATA ...]]
                                 [--eval-config EVAL_CONFIG]
                                 [--eval-input-key EVAL_INPUT_KEY]
                                 [--eval-label-key EVAL_LABEL_KEY]
                                 [--eval-tool-key EVAL_TOOL_KEY]
                                 [--n-samples-per-eval-prompt N_SAMPLES_PER_EVAL_PROMPT]
                                 [--eval-temperature EVAL_TEMPERATURE]
                                 [--eval-top-p EVAL_TOP_P]
                                 [--eval-top-k EVAL_TOP_K]
                                 [--eval-max-response-len EVAL_MAX_RESPONSE_LEN]
                                 [--eval-max-prompt-len EVAL_MAX_PROMPT_LEN]
                                 [--eval-min-new-tokens EVAL_MIN_NEW_TOKENS]
                                 [--eval-max-context-len EVAL_MAX_CONTEXT_LEN]
                                 [--ref-load REF_LOAD]
                                 [--ref-ckpt-step REF_CKPT_STEP] [--load LOAD]
                                 [--save SAVE] [--save-interval SAVE_INTERVAL]
                                 [--seed SEED] [--clip-grad CLIP_GRAD]
                                 [--calculate-per-token-loss]
                                 [--num-critic-only-steps NUM_CRITIC_ONLY_STEPS]
                                 [--critic-load CRITIC_LOAD]
                                 [--critic-save CRITIC_SAVE]
                                 [--critic-lr CRITIC_LR]
                                 [--critic-lr-warmup-iters CRITIC_LR_WARMUP_ITERS]
                                 [--eps-clip EPS_CLIP]
                                 [--eps-clip-high EPS_CLIP_HIGH]
                                 [--eps-clip-c EPS_CLIP_C]
                                 [--value-clip VALUE_CLIP] [--kl-coef KL_COEF]
                                 [--loss-type {policy_loss,sft_loss,custom_loss}]
                                 [--custom-loss-function-path CUSTOM_LOSS_FUNCTION_PATH]
                                 [--kl-loss-type {k1,k2,k3,low_var_kl}]
                                 [--advantage-estimator {grpo,gspo,reinforce_plus_plus,reinforce_plus_plus_baseline,ppo,on_policy_distillation}]
                                 [--disable-compute-advantages-and-returns]
                                 [--use-kl-loss] [--kl-loss-coef KL_LOSS_COEF]
                                 [--use-unbiased-kl]
                                 [--ref-update-interval REF_UPDATE_INTERVAL]
                                 [--entropy-coef ENTROPY_COEF] [--gamma GAMMA]
                                 [--lambd LAMBD] [--normalize-advantages]
                                 [--disable-grpo-std-normalization]
                                 [--disable-rewards-normalization]
                                 [--use-rollout-entropy]
                                 [--get-mismatch-metrics]
                                 [--use-rollout-logprobs] [--use-tis]
                                 [--tis-clip TIS_CLIP]
                                 [--tis-clip-low TIS_CLIP_LOW]
                                 [--custom-tis-function-path CUSTOM_TIS_FUNCTION_PATH]
                                 [--use-routing-replay]
                                 [--use-rollout-routing-replay] [--use-opsm]
                                 [--opsm-delta OPSM_DELTA] [--use-wandb]
                                 [--wandb-mode {online,offline,disabled}]
                                 [--wandb-dir WANDB_DIR]
                                 [--wandb-key WANDB_KEY]
                                 [--wandb-host WANDB_HOST]
                                 [--wandb-team WANDB_TEAM]
                                 [--wandb-group WANDB_GROUP]
                                 [--disable-wandb-random-suffix]
                                 [--wandb-always-use-train-step]
                                 [--log-multi-turn] [--log-passrate]
                                 [--log-reward-category LOG_REWARD_CATEGORY]
                                 [--wandb-run-id WANDB_RUN_ID]
                                 [--use-tensorboard]
                                 [--tb-project-name TB_PROJECT_NAME]
                                 [--tb-experiment-name TB_EXPERIMENT_NAME]
                                 [--use-miles-router]
                                 [--miles-router-middleware-paths MILES_ROUTER_MIDDLEWARE_PATHS [MILES_ROUTER_MIDDLEWARE_PATHS ...]]
                                 [--miles-router-timeout MILES_ROUTER_TIMEOUT]
                                 [--miles-router-max-connections MILES_ROUTER_MAX_CONNECTIONS]
                                 [--worker-urls [WORKER_URLS ...]]
                                 [--router-policy {random,round_robin,cache_aware,power_of_two}]
                                 [--router-prefill-policy {random,round_robin,cache_aware,power_of_two,bucket}]
                                 [--router-decode-policy {random,round_robin,cache_aware,power_of_two}]
                                 [--router-mini-lb]
                                 [--router-pd-disaggregation]
                                 [--router-prefill ROUTER_PREFILL [ROUTER_PREFILL ...]]
                                 [--router-decode URL]
                                 [--router-worker-startup-timeout-secs ROUTER_WORKER_STARTUP_TIMEOUT_SECS]
                                 [--router-worker-startup-check-interval ROUTER_WORKER_STARTUP_CHECK_INTERVAL]
                                 [--router-cache-threshold ROUTER_CACHE_THRESHOLD]
                                 [--router-balance-abs-threshold ROUTER_BALANCE_ABS_THRESHOLD]
                                 [--router-balance-rel-threshold ROUTER_BALANCE_REL_THRESHOLD]
                                 [--router-bucket-adjust-interval-secs ROUTER_BUCKET_ADJUST_INTERVAL_SECS]
                                 [--router-eviction-interval-secs ROUTER_EVICTION_INTERVAL_SECS]
                                 [--router-max-tree-size ROUTER_MAX_TREE_SIZE]
                                 [--router-max-payload-size ROUTER_MAX_PAYLOAD_SIZE]
                                 [--router-dp-aware] [--router-enable-igw]
                                 [--router-api-key ROUTER_API_KEY]
                                 [--router-log-dir ROUTER_LOG_DIR]
                                 [--router-log-level {debug,info,warn,error}]
                                 [--router-service-discovery]
                                 [--router-selector ROUTER_SELECTOR [ROUTER_SELECTOR ...]]
                                 [--router-service-discovery-port ROUTER_SERVICE_DISCOVERY_PORT]
                                 [--router-service-discovery-namespace ROUTER_SERVICE_DISCOVERY_NAMESPACE]
                                 [--router-prefill-selector ROUTER_PREFILL_SELECTOR [ROUTER_PREFILL_SELECTOR ...]]
                                 [--router-decode-selector ROUTER_DECODE_SELECTOR [ROUTER_DECODE_SELECTOR ...]]
                                 [--router-prometheus-port ROUTER_PROMETHEUS_PORT]
                                 [--router-prometheus-host ROUTER_PROMETHEUS_HOST]
                                 [--router-request-id-headers [ROUTER_REQUEST_ID_HEADERS ...]]
                                 [--router-request-timeout-secs ROUTER_REQUEST_TIMEOUT_SECS]
                                 [--router-retry-max-retries ROUTER_RETRY_MAX_RETRIES]
                                 [--router-retry-initial-backoff-ms ROUTER_RETRY_INITIAL_BACKOFF_MS]
                                 [--router-retry-max-backoff-ms ROUTER_RETRY_MAX_BACKOFF_MS]
                                 [--router-retry-backoff-multiplier ROUTER_RETRY_BACKOFF_MULTIPLIER]
                                 [--router-retry-jitter-factor ROUTER_RETRY_JITTER_FACTOR]
                                 [--router-disable-retries]
                                 [--router-cb-failure-threshold ROUTER_CB_FAILURE_THRESHOLD]
                                 [--router-cb-success-threshold ROUTER_CB_SUCCESS_THRESHOLD]
                                 [--router-cb-timeout-duration-secs ROUTER_CB_TIMEOUT_DURATION_SECS]
                                 [--router-cb-window-duration-secs ROUTER_CB_WINDOW_DURATION_SECS]
                                 [--router-disable-circuit-breaker]
                                 [--router-health-failure-threshold ROUTER_HEALTH_FAILURE_THRESHOLD]
                                 [--router-health-success-threshold ROUTER_HEALTH_SUCCESS_THRESHOLD]
                                 [--router-health-check-timeout-secs ROUTER_HEALTH_CHECK_TIMEOUT_SECS]
                                 [--router-health-check-interval-secs ROUTER_HEALTH_CHECK_INTERVAL_SECS]
                                 [--router-health-check-endpoint ROUTER_HEALTH_CHECK_ENDPOINT]
                                 [--router-max-concurrent-requests ROUTER_MAX_CONCURRENT_REQUESTS]
                                 [--router-queue-size ROUTER_QUEUE_SIZE]
                                 [--router-queue-timeout-secs ROUTER_QUEUE_TIMEOUT_SECS]
                                 [--router-rate-limit-tokens-per-second ROUTER_RATE_LIMIT_TOKENS_PER_SECOND]
                                 [--router-cors-allowed-origins [ROUTER_CORS_ALLOWED_ORIGINS ...]]
                                 [--router-model-path ROUTER_MODEL_PATH]
                                 [--router-tokenizer-path ROUTER_TOKENIZER_PATH]
                                 [--router-chat-template ROUTER_CHAT_TEMPLATE]
                                 [--router-tokenizer-cache-enable-l0]
                                 [--router-tokenizer-cache-l0-max-entries ROUTER_TOKENIZER_CACHE_L0_MAX_ENTRIES]
                                 [--router-tokenizer-cache-enable-l1]
                                 [--router-tokenizer-cache-l1-max-memory ROUTER_TOKENIZER_CACHE_L1_MAX_MEMORY]
                                 [--router-reasoning-parser ROUTER_REASONING_PARSER]
                                 [--router-tool-call-parser ROUTER_TOOL_CALL_PARSER]
                                 [--router-mcp-config-path ROUTER_MCP_CONFIG_PATH]
                                 [--router-backend {sglang,openai}]
                                 [--router-history-backend {memory,none,oracle,postgres}]
                                 [--router-oracle-wallet-path ROUTER_ORACLE_WALLET_PATH]
                                 [--router-oracle-tns-alias ROUTER_ORACLE_TNS_ALIAS]
                                 [--router-oracle-connect-descriptor ROUTER_ORACLE_CONNECT_DESCRIPTOR]
                                 [--router-oracle-username ROUTER_ORACLE_USERNAME]
                                 [--router-oracle-password ROUTER_ORACLE_PASSWORD]
                                 [--router-oracle-pool-min ROUTER_ORACLE_POOL_MIN]
                                 [--router-oracle-pool-max ROUTER_ORACLE_POOL_MAX]
                                 [--router-oracle-pool-timeout-secs ROUTER_ORACLE_POOL_TIMEOUT_SECS]
                                 [--router-postgres-db-url ROUTER_POSTGRES_DB_URL]
                                 [--router-postgres-pool-max ROUTER_POSTGRES_POOL_MAX]
                                 [--router-client-cert-path ROUTER_CLIENT_CERT_PATH]
                                 [--router-client-key-path ROUTER_CLIENT_KEY_PATH]
                                 [--router-ca-cert-paths [ROUTER_CA_CERT_PATHS ...]]
                                 [--router-enable-trace]
                                 [--router-otlp-traces-endpoint ROUTER_OTLP_TRACES_ENDPOINT]
                                 [--save-debug-rollout-data SAVE_DEBUG_ROLLOUT_DATA]
                                 [--load-debug-rollout-data LOAD_DEBUG_ROLLOUT_DATA]
                                 [--load-debug-rollout-data-subsample LOAD_DEBUG_ROLLOUT_DATA_SUBSAMPLE]
                                 [--debug-rollout-only] [--debug-train-only]
                                 [--save-debug-train-data SAVE_DEBUG_TRAIN_DATA]
                                 [--dump-details DUMP_DETAILS]
                                 [--memory-snapshot-dir MEMORY_SNAPSHOT_DIR]
                                 [--memory-snapshot-num-steps MEMORY_SNAPSHOT_NUM_STEPS]
                                 [--profile-target {train_overall,train_actor,train_log_probs} [{train_overall,train_actor,train_log_probs} ...]]
                                 [--memory-recorder {torch,memray}]
                                 [--check-weight-update-equal]
                                 [--sglang-router-ip SGLANG_ROUTER_IP]
                                 [--sglang-router-port SGLANG_ROUTER_PORT]
                                 [--sglang-router-request-timeout-secs SGLANG_ROUTER_REQUEST_TIMEOUT_SECS]
                                 [--sglang-server-concurrency SGLANG_SERVER_CONCURRENCY]
                                 [--sglang-tokenizer-path SGLANG_TOKENIZER_PATH]
                                 [--sglang-tokenizer-mode {auto,slow}]
                                 [--sglang-tokenizer-worker-num SGLANG_TOKENIZER_WORKER_NUM]
                                 [--sglang-skip-tokenizer-init]
                                 [--sglang-load-format {auto,pt,safetensors,npcache,dummy,sharded_state,gguf,bitsandbytes,layered,remote,remote_instance}]
                                 [--sglang-model-loader-extra-config SGLANG_MODEL_LOADER_EXTRA_CONFIG]
                                 [--sglang-context-length SGLANG_CONTEXT_LENGTH]
                                 [--sglang-is-embedding]
                                 [--sglang-enable-multimodal]
                                 [--sglang-revision SGLANG_REVISION]
                                 [--sglang-model-impl SGLANG_MODEL_IMPL]
                                 [--sglang-host SGLANG_HOST]
                                 [--sglang-grpc-mode]
                                 [--sglang-warmups SGLANG_WARMUPS]
                                 [--sglang-checkpoint-engine-wait-weights-before-ready]
                                 [--sglang-quantization {awq,fp8,gptq,marlin,gptq_marlin,awq_marlin,bitsandbytes,gguf,modelopt,modelopt_fp8,modelopt_fp4,petit_nvfp4,w8a8_int8,w8a8_fp8,moe_wna16,qoq,w4afp8,mxfp4,auto-round,compressed-tensors}]
                                 [--sglang-quantization-param-path SGLANG_QUANTIZATION_PARAM_PATH]
                                 [--sglang-kv-cache-dtype {auto,fp8_e5m2,fp8_e4m3,bf16,bfloat16,fp4_e2m1}]
                                 [--sglang-enable-fp32-lm-head]
                                 [--sglang-modelopt-quant SGLANG_MODELOPT_QUANT]
                                 [--sglang-modelopt-checkpoint-restore-path SGLANG_MODELOPT_CHECKPOINT_RESTORE_PATH]
                                 [--sglang-modelopt-checkpoint-save-path SGLANG_MODELOPT_CHECKPOINT_SAVE_PATH]
                                 [--sglang-modelopt-export-path SGLANG_MODELOPT_EXPORT_PATH]
                                 [--sglang-quantize-and-serve]
                                 [--sglang-mem-fraction-static SGLANG_MEM_FRACTION_STATIC]
                                 [--sglang-max-running-requests SGLANG_MAX_RUNNING_REQUESTS]
                                 [--sglang-max-queued-requests SGLANG_MAX_QUEUED_REQUESTS]
                                 [--sglang-max-total-tokens SGLANG_MAX_TOTAL_TOKENS]
                                 [--sglang-chunked-prefill-size SGLANG_CHUNKED_PREFILL_SIZE]
                                 [--sglang-max-prefill-tokens SGLANG_MAX_PREFILL_TOKENS]
                                 [--sglang-schedule-policy {lpm,random,fcfs,dfs-weight,lof,priority}]
                                 [--sglang-enable-priority-scheduling]
                                 [--sglang-abort-on-priority-when-disabled]
                                 [--sglang-schedule-low-priority-values-first]
                                 [--sglang-priority-scheduling-preemption-threshold SGLANG_PRIORITY_SCHEDULING_PREEMPTION_THRESHOLD]
                                 [--sglang-schedule-conservativeness SGLANG_SCHEDULE_CONSERVATIVENESS]
                                 [--sglang-page-size SGLANG_PAGE_SIZE]
                                 [--sglang-hybrid-kvcache-ratio [SGLANG_HYBRID_KVCACHE_RATIO]]
                                 [--sglang-swa-full-tokens-ratio SGLANG_SWA_FULL_TOKENS_RATIO]
                                 [--sglang-disable-hybrid-swa-memory]
                                 [--sglang-radix-eviction-policy {lru,lfu}]
                                 [--sglang-device SGLANG_DEVICE]
                                 [--sglang-tensor-parallel-size SGLANG_TENSOR_PARALLEL_SIZE]
                                 [--sglang-pipeline-parallel-size SGLANG_PIPELINE_PARALLEL_SIZE]
                                 [--sglang-pp-max-micro-batch-size SGLANG_PP_MAX_MICRO_BATCH_SIZE]
                                 [--sglang-stream-interval SGLANG_STREAM_INTERVAL]
                                 [--sglang-stream-output]
                                 [--sglang-constrained-json-whitespace-pattern SGLANG_CONSTRAINED_JSON_WHITESPACE_PATTERN]
                                 [--sglang-constrained-json-disable-any-whitespace]
                                 [--sglang-watchdog-timeout SGLANG_WATCHDOG_TIMEOUT]
                                 [--sglang-dist-timeout SGLANG_DIST_TIMEOUT]
                                 [--sglang-download-dir SGLANG_DOWNLOAD_DIR]
                                 [--sglang-sleep-on-idle]
                                 [--sglang-log-level SGLANG_LOG_LEVEL]
                                 [--sglang-log-level-http SGLANG_LOG_LEVEL_HTTP]
                                 [--sglang-log-requests]
                                 [--sglang-log-requests-level {0,1,2,3}]
                                 [--sglang-crash-dump-folder SGLANG_CRASH_DUMP_FOLDER]
                                 [--sglang-show-time-cost]
                                 [--sglang-enable-metrics]
                                 [--sglang-enable-metrics-for-all-schedulers]
                                 [--sglang-tokenizer-metrics-custom-labels-header SGLANG_TOKENIZER_METRICS_CUSTOM_LABELS_HEADER]
                                 [--sglang-tokenizer-metrics-allowed-custom-labels SGLANG_TOKENIZER_METRICS_ALLOWED_CUSTOM_LABELS [SGLANG_TOKENIZER_METRICS_ALLOWED_CUSTOM_LABELS ...]]
                                 [--sglang-bucket-time-to-first-token SGLANG_BUCKET_TIME_TO_FIRST_TOKEN [SGLANG_BUCKET_TIME_TO_FIRST_TOKEN ...]]
                                 [--sglang-bucket-inter-token-latency SGLANG_BUCKET_INTER_TOKEN_LATENCY [SGLANG_BUCKET_INTER_TOKEN_LATENCY ...]]
                                 [--sglang-bucket-e2e-request-latency SGLANG_BUCKET_E2E_REQUEST_LATENCY [SGLANG_BUCKET_E2E_REQUEST_LATENCY ...]]
                                 [--sglang-collect-tokens-histogram]
                                 [--sglang-prompt-tokens-buckets SGLANG_PROMPT_TOKENS_BUCKETS [SGLANG_PROMPT_TOKENS_BUCKETS ...]]
                                 [--sglang-generation-tokens-buckets SGLANG_GENERATION_TOKENS_BUCKETS [SGLANG_GENERATION_TOKENS_BUCKETS ...]]
                                 [--sglang-gc-warning-threshold-secs SGLANG_GC_WARNING_THRESHOLD_SECS]
                                 [--sglang-decode-log-interval SGLANG_DECODE_LOG_INTERVAL]
                                 [--sglang-enable-request-time-stats-logging]
                                 [--sglang-kv-events-config SGLANG_KV_EVENTS_CONFIG]
                                 [--sglang-enable-trace]
                                 [--sglang-otlp-traces-endpoint SGLANG_OTLP_TRACES_ENDPOINT]
                                 [--sglang-api-key SGLANG_API_KEY]
                                 [--sglang-served-model-name SGLANG_SERVED_MODEL_NAME]
                                 [--sglang-weight-version SGLANG_WEIGHT_VERSION]
                                 [--sglang-chat-template SGLANG_CHAT_TEMPLATE]
                                 [--sglang-completion-template SGLANG_COMPLETION_TEMPLATE]
                                 [--sglang-file-storage-path SGLANG_FILE_STORAGE_PATH]
                                 [--sglang-enable-cache-report]
                                 [--sglang-reasoning-parser {deepseek-r1,deepseek-v3,glm45,gpt-oss,kimi,kimi_k2,qwen3,qwen3-thinking,minimax,minimax-append-think,step3}]
                                 [--sglang-tool-call-parser {deepseekv3,deepseekv31,glm,glm45,gpt-oss,kimi_k2,llama3,mistral,pythonic,qwen,qwen25,qwen3_coder,step3,minimax-m2}]
                                 [--sglang-tool-server SGLANG_TOOL_SERVER]
                                 [--sglang-sampling-defaults {openai,model}]
                                 [--sglang-data-parallel-size SGLANG_DATA_PARALLEL_SIZE]
                                 [--sglang-load-balance-method {round_robin,shortest_queue,minimum_tokens}]
                                 [--sglang-load-watch-interval SGLANG_LOAD_WATCH_INTERVAL]
                                 [--sglang-prefill-round-robin-balance]
                                 [--sglang-json-model-override-args SGLANG_JSON_MODEL_OVERRIDE_ARGS]
                                 [--sglang-preferred-sampling-params SGLANG_PREFERRED_SAMPLING_PARAMS]
                                 [--sglang-enable-lora]
                                 [--sglang-max-lora-rank SGLANG_MAX_LORA_RANK]
                                 [--sglang-lora-target-modules [{q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj,qkv_proj,gate_up_proj,all} ...]]
                                 [--sglang-lora-paths [SGLANG_LORA_PATHS ...]]
                                 [--sglang-max-loras-per-batch SGLANG_MAX_LORAS_PER_BATCH]
                                 [--sglang-max-loaded-loras SGLANG_MAX_LOADED_LORAS]
                                 [--sglang-lora-eviction-policy {lru,fifo}]
                                 [--sglang-lora-backend {triton,csgmv}]
                                 [--sglang-max-lora-chunk-size {16,32,64,128}]
                                 [--sglang-attention-backend {triton,torch_native,flex_attention,nsa,cutlass_mla,fa3,fa4,flashinfer,flashmla,trtllm_mla,trtllm_mha,dual_chunk_flash_attn,aiter,wave,intel_amx,ascend,intel_xpu}]
                                 [--sglang-prefill-attention-backend {triton,torch_native,flex_attention,nsa,cutlass_mla,fa3,fa4,flashinfer,flashmla,trtllm_mla,trtllm_mha,dual_chunk_flash_attn,aiter,wave,intel_amx,ascend,intel_xpu}]
                                 [--sglang-decode-attention-backend {triton,torch_native,flex_attention,nsa,cutlass_mla,fa3,fa4,flashinfer,flashmla,trtllm_mla,trtllm_mha,dual_chunk_flash_attn,aiter,wave,intel_amx,ascend,intel_xpu}]
                                 [--sglang-sampling-backend {flashinfer,pytorch}]
                                 [--sglang-grammar-backend {xgrammar,outlines,llguidance,none}]
                                 [--sglang-mm-attention-backend {sdpa,fa3,triton_attn,ascend_attn,aiter_attn}]
                                 [--sglang-nsa-prefill-backend {flashmla_sparse,flashmla_kv,flashmla_auto,fa3,tilelang,aiter}]
                                 [--sglang-nsa-decode-backend {flashmla_sparse,flashmla_kv,flashmla_auto,fa3,tilelang,aiter}]
                                 [--sglang-speculative-algorithm {EAGLE,EAGLE3,NEXTN,STANDALONE,NGRAM}]
                                 [--sglang-speculative-draft-model-path SGLANG_SPECULATIVE_DRAFT_MODEL_PATH]
                                 [--sglang-speculative-draft-model-revision SGLANG_SPECULATIVE_DRAFT_MODEL_REVISION]
                                 [--sglang-speculative-draft-load-format {auto,pt,safetensors,npcache,dummy,sharded_state,gguf,bitsandbytes,layered,remote,remote_instance}]
                                 [--sglang-speculative-num-steps SGLANG_SPECULATIVE_NUM_STEPS]
                                 [--sglang-speculative-eagle-topk SGLANG_SPECULATIVE_EAGLE_TOPK]
                                 [--sglang-speculative-num-draft-tokens SGLANG_SPECULATIVE_NUM_DRAFT_TOKENS]
                                 [--sglang-speculative-accept-threshold-single SGLANG_SPECULATIVE_ACCEPT_THRESHOLD_SINGLE]
                                 [--sglang-speculative-accept-threshold-acc SGLANG_SPECULATIVE_ACCEPT_THRESHOLD_ACC]
                                 [--sglang-speculative-token-map SGLANG_SPECULATIVE_TOKEN_MAP]
                                 [--sglang-speculative-attention-mode {prefill,decode}]
                                 [--sglang-speculative-moe-runner-backend {auto,deep_gemm,triton,triton_kernel,flashinfer_trtllm,flashinfer_cutlass,flashinfer_mxfp4,flashinfer_cutedsl,cutlass}]
                                 [--sglang-speculative-ngram-min-match-window-size SGLANG_SPECULATIVE_NGRAM_MIN_MATCH_WINDOW_SIZE]
                                 [--sglang-speculative-ngram-max-match-window-size SGLANG_SPECULATIVE_NGRAM_MAX_MATCH_WINDOW_SIZE]
                                 [--sglang-speculative-ngram-min-bfs-breadth SGLANG_SPECULATIVE_NGRAM_MIN_BFS_BREADTH]
                                 [--sglang-speculative-ngram-max-bfs-breadth SGLANG_SPECULATIVE_NGRAM_MAX_BFS_BREADTH]
                                 [--sglang-speculative-ngram-match-type {BFS,PROB}]
                                 [--sglang-speculative-ngram-branch-length SGLANG_SPECULATIVE_NGRAM_BRANCH_LENGTH]
                                 [--sglang-speculative-ngram-capacity SGLANG_SPECULATIVE_NGRAM_CAPACITY]
                                 [--sglang-expert-parallel-size SGLANG_EXPERT_PARALLEL_SIZE]
                                 [--sglang-moe-a2a-backend {none,deepep,mooncake}]
                                 [--sglang-moe-runner-backend {auto,deep_gemm,triton,triton_kernel,flashinfer_trtllm,flashinfer_cutlass,flashinfer_mxfp4,flashinfer_cutedsl,cutlass}]
                                 [--sglang-flashinfer-mxfp4-moe-precision {default,bf16}]
                                 [--sglang-enable-flashinfer-allreduce-fusion]
                                 [--sglang-deepep-mode {normal,low_latency,auto}]
                                 [--sglang-ep-num-redundant-experts SGLANG_EP_NUM_REDUNDANT_EXPERTS]
                                 [--sglang-ep-dispatch-algorithm SGLANG_EP_DISPATCH_ALGORITHM]
                                 [--sglang-init-expert-location SGLANG_INIT_EXPERT_LOCATION]
                                 [--sglang-enable-eplb]
                                 [--sglang-eplb-algorithm SGLANG_EPLB_ALGORITHM]
                                 [--sglang-eplb-rebalance-num-iterations SGLANG_EPLB_REBALANCE_NUM_ITERATIONS]
                                 [--sglang-eplb-rebalance-layers-per-chunk SGLANG_EPLB_REBALANCE_LAYERS_PER_CHUNK]
                                 [--sglang-eplb-min-rebalancing-utilization-threshold SGLANG_EPLB_MIN_REBALANCING_UTILIZATION_THRESHOLD]
                                 [--sglang-expert-distribution-recorder-mode SGLANG_EXPERT_DISTRIBUTION_RECORDER_MODE]
                                 [--sglang-expert-distribution-recorder-buffer-size SGLANG_EXPERT_DISTRIBUTION_RECORDER_BUFFER_SIZE]
                                 [--sglang-enable-expert-distribution-metrics]
                                 [--sglang-deepep-config SGLANG_DEEPEP_CONFIG]
                                 [--sglang-moe-dense-tp-size SGLANG_MOE_DENSE_TP_SIZE]
                                 [--sglang-elastic-ep-backend {none,mooncake}]
                                 [--sglang-mooncake-ib-device SGLANG_MOONCAKE_IB_DEVICE]
                                 [--sglang-max-mamba-cache-size SGLANG_MAX_MAMBA_CACHE_SIZE]
                                 [--sglang-mamba-ssm-dtype {float32,bfloat16}]
                                 [--sglang-mamba-full-memory-ratio SGLANG_MAMBA_FULL_MEMORY_RATIO]
                                 [--sglang-enable-hierarchical-cache]
                                 [--sglang-hicache-ratio SGLANG_HICACHE_RATIO]
                                 [--sglang-hicache-size SGLANG_HICACHE_SIZE]
                                 [--sglang-hicache-write-policy {write_back,write_through,write_through_selective}]
                                 [--sglang-hicache-io-backend {direct,kernel}]
                                 [--sglang-hicache-mem-layout {layer_first,page_first,page_first_direct}]
                                 [--sglang-hicache-storage-backend {file,mooncake,hf3fs,nixl,aibrix,dynamic,eic}]
                                 [--sglang-hicache-storage-prefetch-policy {best_effort,wait_complete,timeout}]
                                 [--sglang-hicache-storage-backend-extra-config SGLANG_HICACHE_STORAGE_BACKEND_EXTRA_CONFIG]
                                 [--sglang-enable-lmcache]
                                 [--sglang-kt-weight-path SGLANG_KT_WEIGHT_PATH]
                                 [--sglang-kt-method SGLANG_KT_METHOD]
                                 [--sglang-kt-cpuinfer SGLANG_KT_CPUINFER]
                                 [--sglang-kt-threadpool-count SGLANG_KT_THREADPOOL_COUNT]
                                 [--sglang-kt-num-gpu-experts SGLANG_KT_NUM_GPU_EXPERTS]
                                 [--sglang-kt-max-deferred-experts-per-token SGLANG_KT_MAX_DEFERRED_EXPERTS_PER_TOKEN]
                                 [--sglang-enable-double-sparsity]
                                 [--sglang-ds-channel-config-path SGLANG_DS_CHANNEL_CONFIG_PATH]
                                 [--sglang-ds-heavy-channel-num SGLANG_DS_HEAVY_CHANNEL_NUM]
                                 [--sglang-ds-heavy-token-num SGLANG_DS_HEAVY_TOKEN_NUM]
                                 [--sglang-ds-heavy-channel-type SGLANG_DS_HEAVY_CHANNEL_TYPE]
                                 [--sglang-ds-sparse-decode-threshold SGLANG_DS_SPARSE_DECODE_THRESHOLD]
                                 [--sglang-cpu-offload-gb SGLANG_CPU_OFFLOAD_GB]
                                 [--sglang-offload-group-size SGLANG_OFFLOAD_GROUP_SIZE]
                                 [--sglang-offload-num-in-group SGLANG_OFFLOAD_NUM_IN_GROUP]
                                 [--sglang-offload-prefetch-step SGLANG_OFFLOAD_PREFETCH_STEP]
                                 [--sglang-offload-mode SGLANG_OFFLOAD_MODE]
                                 [--sglang-multi-item-scoring-delimiter SGLANG_MULTI_ITEM_SCORING_DELIMITER]
                                 [--sglang-disable-radix-cache]
                                 [--sglang-cuda-graph-max-bs SGLANG_CUDA_GRAPH_MAX_BS]
                                 [--sglang-cuda-graph-bs SGLANG_CUDA_GRAPH_BS [SGLANG_CUDA_GRAPH_BS ...]]
                                 [--sglang-disable-cuda-graph]
                                 [--sglang-disable-cuda-graph-padding]
                                 [--sglang-enable-profile-cuda-graph]
                                 [--sglang-enable-cudagraph-gc]
                                 [--sglang-enable-nccl-nvls]
                                 [--sglang-enable-symm-mem]
                                 [--sglang-disable-flashinfer-cutlass-moe-fp4-allgather]
                                 [--sglang-enable-tokenizer-batch-encode]
                                 [--sglang-disable-tokenizer-batch-decode]
                                 [--sglang-disable-outlines-disk-cache]
                                 [--sglang-disable-custom-all-reduce]
                                 [--sglang-enable-mscclpp]
                                 [--sglang-enable-torch-symm-mem]
                                 [--sglang-disable-overlap-schedule]
                                 [--sglang-enable-mixed-chunk]
                                 [--sglang-enable-dp-attention]
                                 [--sglang-enable-dp-lm-head]
                                 [--sglang-enable-two-batch-overlap]
                                 [--sglang-enable-single-batch-overlap]
                                 [--sglang-tbo-token-distribution-threshold SGLANG_TBO_TOKEN_DISTRIBUTION_THRESHOLD]
                                 [--sglang-enable-torch-compile]
                                 [--sglang-enable-piecewise-cuda-graph]
                                 [--sglang-piecewise-cuda-graph-tokens SGLANG_PIECEWISE_CUDA_GRAPH_TOKENS]
                                 [--sglang-piecewise-cuda-graph-compiler {eager,inductor}]
                                 [--sglang-torch-compile-max-bs SGLANG_TORCH_COMPILE_MAX_BS]
                                 [--sglang-piecewise-cuda-graph-max-tokens SGLANG_PIECEWISE_CUDA_GRAPH_MAX_TOKENS]
                                 [--sglang-torchao-config SGLANG_TORCHAO_CONFIG]
                                 [--sglang-enable-nan-detection]
                                 [--sglang-enable-p2p-check]
                                 [--sglang-triton-attention-reduce-in-fp32]
                                 [--sglang-triton-attention-num-kv-splits SGLANG_TRITON_ATTENTION_NUM_KV_SPLITS]
                                 [--sglang-triton-attention-split-tile-size SGLANG_TRITON_ATTENTION_SPLIT_TILE_SIZE]
                                 [--sglang-num-continuous-decode-steps SGLANG_NUM_CONTINUOUS_DECODE_STEPS]
                                 [--sglang-delete-ckpt-after-loading]
                                 [--sglang-enable-weights-cpu-backup]
                                 [--sglang-allow-auto-truncate]
                                 [--sglang-enable-custom-logit-processor]
                                 [--sglang-flashinfer-mla-disable-ragged]
                                 [--sglang-disable-shared-experts-fusion]
                                 [--sglang-disable-chunked-prefix-cache]
                                 [--sglang-disable-fast-image-processor]
                                 [--sglang-keep-mm-feature-on-device]
                                 [--sglang-enable-return-hidden-states]
                                 [--sglang-scheduler-recv-interval SGLANG_SCHEDULER_RECV_INTERVAL]
                                 [--sglang-numa-node SGLANG_NUMA_NODE [SGLANG_NUMA_NODE ...]]
                                 [--sglang-enable-deterministic-inference]
                                 [--sglang-rl-on-policy-target {fsdp}]
                                 [--sglang-enable-dynamic-batch-tokenizer]
                                 [--sglang-dynamic-batch-tokenizer-batch-size SGLANG_DYNAMIC_BATCH_TOKENIZER_BATCH_SIZE]
                                 [--sglang-dynamic-batch-tokenizer-batch-timeout SGLANG_DYNAMIC_BATCH_TOKENIZER_BATCH_TIMEOUT]
                                 [--sglang-debug-tensor-dump-output-folder SGLANG_DEBUG_TENSOR_DUMP_OUTPUT_FOLDER]
                                 [--sglang-debug-tensor-dump-layers SGLANG_DEBUG_TENSOR_DUMP_LAYERS [SGLANG_DEBUG_TENSOR_DUMP_LAYERS ...]]
                                 [--sglang-debug-tensor-dump-input-file SGLANG_DEBUG_TENSOR_DUMP_INPUT_FILE]
                                 [--sglang-debug-tensor-dump-inject SGLANG_DEBUG_TENSOR_DUMP_INJECT]
                                 [--sglang-disaggregation-mode {null,prefill,decode}]
                                 [--sglang-disaggregation-transfer-backend {mooncake,nixl,ascend,fake}]
                                 [--sglang-disaggregation-bootstrap-port SGLANG_DISAGGREGATION_BOOTSTRAP_PORT]
                                 [--sglang-disaggregation-decode-tp SGLANG_DISAGGREGATION_DECODE_TP]
                                 [--sglang-disaggregation-decode-dp SGLANG_DISAGGREGATION_DECODE_DP]
                                 [--sglang-disaggregation-prefill-pp SGLANG_DISAGGREGATION_PREFILL_PP]
                                 [--sglang-disaggregation-ib-device SGLANG_DISAGGREGATION_IB_DEVICE]
                                 [--sglang-disaggregation-decode-enable-offload-kvcache]
                                 [--sglang-num-reserved-decode-tokens SGLANG_NUM_RESERVED_DECODE_TOKENS]
                                 [--sglang-disaggregation-decode-polling-interval SGLANG_DISAGGREGATION_DECODE_POLLING_INTERVAL]
                                 [--sglang-custom-weight-loader [SGLANG_CUSTOM_WEIGHT_LOADER ...]]
                                 [--sglang-weight-loader-disable-mmap]
                                 [--sglang-remote-instance-weight-loader-seed-instance-ip SGLANG_REMOTE_INSTANCE_WEIGHT_LOADER_SEED_INSTANCE_IP]
                                 [--sglang-remote-instance-weight-loader-seed-instance-service-port SGLANG_REMOTE_INSTANCE_WEIGHT_LOADER_SEED_INSTANCE_SERVICE_PORT]
                                 [--sglang-remote-instance-weight-loader-send-weights-group-ports SGLANG_REMOTE_INSTANCE_WEIGHT_LOADER_SEND_WEIGHTS_GROUP_PORTS]
                                 [--sglang-enable-pdmux]
                                 [--sglang-pdmux-config-path SGLANG_PDMUX_CONFIG_PATH]
                                 [--sglang-sm-group-num SGLANG_SM_GROUP_NUM]
                                 [--sglang-config SGLANG_CONFIG]
                                 [--sglang-mm-max-concurrent-calls SGLANG_MM_MAX_CONCURRENT_CALLS]
                                 [--sglang-mm-per-request-timeout SGLANG_MM_PER_REQUEST_TIMEOUT]
                                 [--sglang-decrypted-config-file SGLANG_DECRYPTED_CONFIG_FILE]
                                 [--sglang-decrypted-draft-config-file SGLANG_DECRYPTED_DRAFT_CONFIG_FILE]
                                 [--http-proxy HTTP_PROXY]
                                 [--use-distributed-post] [--rm-type RM_TYPE]
                                 [--reward-key REWARD_KEY]
                                 [--eval-reward-key EVAL_REWARD_KEY]
                                 [--group-rm] [--rm-url RM_URL]
                                 [--custom-rm-path CUSTOM_RM_PATH]
                                 [--custom-reward-post-process-path CUSTOM_REWARD_POST_PROCESS_PATH]
                                 [--rollout-buffer-url ROLLOUT_BUFFER_URL]
                                 [--fetch-trajectory-retry-times FETCH_TRAJECTORY_RETRY_TIMES]
                                 [--min-batch-collection-ratio MIN_BATCH_COLLECTION_RATIO]
                                 [--rollout-task-type ROLLOUT_TASK_TYPE]
                                 [--loss-mask-type {qwen,qwen3,distill_qwen}]
                                 [--data-pad-size-multiplier DATA_PAD_SIZE_MULTIPLIER]
                                 [--rollout-sample-filter-path ROLLOUT_SAMPLE_FILTER_PATH]
                                 [--disable-rollout-trim-samples]
                                 [--mtp-num-layers MTP_NUM_LAYERS]
                                 [--mtp-loss-scaling-factor MTP_LOSS_SCALING_FACTOR]
                                 [--enable-mtp-training]
                                 [--prefill-num-servers PREFILL_NUM_SERVERS]
                                 [--ci-test] [--ci-disable-kl-checker]
                                 [--ci-metric-checker-key CI_METRIC_CHECKER_KEY]
                                 [--ci-metric-checker-threshold CI_METRIC_CHECKER_THRESHOLD]
                                 [--ci-save-grad-norm CI_SAVE_GRAD_NORM]
                                 [--ci-load-grad-norm CI_LOAD_GRAD_NORM]
                                 [--custom-megatron-init-path CUSTOM_MEGATRON_INIT_PATH]
                                 [--custom-megatron-before-log-prob-hook-path CUSTOM_MEGATRON_BEFORE_LOG_PROB_HOOK_PATH]
                                 [--custom-megatron-before-train-step-hook-path CUSTOM_MEGATRON_BEFORE_TRAIN_STEP_HOOK_PATH]
                                 [--custom-config-path CUSTOM_CONFIG_PATH]
                                 [--padded-vocab-size PADDED_VOCAB_SIZE]
FSDP SFT Training (miles): error: unrecognized arguments: --lr_decay_iters 1900
2025-12-13 15:21:37,638	ERR cli.py:73 -- ---------------------------------------
2025-12-13 15:21:37,638	ERR cli.py:74 -- Job 'raysubmit_MST4XU4FXkhk8Csq' failed
2025-12-13 15:21:37,638	ERR cli.py:75 -- ---------------------------------------
2025-12-13 15:21:37,638	INFO cli.py:88 -- Status message: Job entrypoint command failed with exit code 2, last available logs (truncated to 20,000 chars):
                                 [--ci-metric-checker-key CI_METRIC_CHECKER_KEY]
                                 [--ci-metric-checker-threshold CI_METRIC_CHECKER_THRESHOLD]
                                 [--ci-save-grad-norm CI_SAVE_GRAD_NORM]
                                 [--ci-load-grad-norm CI_LOAD_GRAD_NORM]
                                 [--custom-megatron-init-path CUSTOM_MEGATRON_INIT_PATH]
                                 [--custom-megatron-before-log-prob-hook-path CUSTOM_MEGATRON_BEFORE_LOG_PROB_HOOK_PATH]
                                 [--custom-megatron-before-train-step-hook-path CUSTOM_MEGATRON_BEFORE_TRAIN_STEP_HOOK_PATH]
                                 [--custom-config-path CUSTOM_CONFIG_PATH]
                                 [--padded-vocab-size PADDED_VOCAB_SIZE]
FSDP SFT Training (miles): error: unrecognized arguments: --lr_decay_iters 1900

srun: error: hai004: task 0: Exited with exit code 1