[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/nemo/qwen3_600m_finetune_requeue_34948.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=00:30:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune_requeue
#SBATCH --mem=400GB
#SBATCH --qos=low
#SBATCH --requeue
#SBATCH --signal=b:usr1@60 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

srun python slurm/dev/mihir/berlin/crowd_pilot/nemo/iterating_on_stuff/test_full_finetune_4k_requeue.py &

child_pid=$!

wait $child_pid
/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
─ Entering Experiment qwen3_600m_full_4k_tokens_34940 with id: qwen3_600m_ful… ─
[16:07:03] INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_full_4k_tokens_34940/                       
                    qwen3_600m_full_4k_tokens_34940_17642                       
                    56023/qwen3_600m_full_4k_tokens_34940                       
[16:07:03] Launching job qwen3_600m_full_4k_tokens_34940 for   experiment.py:744
           experiment qwen3_600m_full_4k_tokens_34940                           
           INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_full_4k_tokens_34940/                       
                    qwen3_600m_full_4k_tokens_34940_17642                       
                    56023/qwen3_600m_full_4k_tokens_34940                       
           INFO     Launched app:                                launcher.py:111
                    local_persistent://nemo_run/qwen3_600m_full_                
                    4k_tokens_34940-wrxhmhjfnd12dc                              
─ Waiting for Experiment qwen3_600m_full_4k_tokens_34940_1764256023 to finish ──

Experiment Status for qwen3_600m_full_4k_tokens_34940_1764256023

Task 0: qwen3_600m_full_4k_tokens_34940
- Status: RUNNING
- Executor: LocalExecutor
- Job id: qwen3_600m_full_4k_tokens_34940-wrxhmhjfnd12dc
- Local Directory: /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_full_4k_tokens_34940/qwen3_600m_full_4k_tokens_34940_1764256023/qwen3_600m_full_4k_tokens_34940

           INFO     Waiting for job                              launcher.py:130
                    qwen3_600m_full_4k_tokens_34940-wrxhmhjfnd12                
                    dc to finish [log=True]...                                  
kens_34940/0 /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
kens_34940/0   import pynvml  # type: ignore[import]
kens_34940/0 I1127 16:07:06.257000 2417549 torch/distributed/run.py:657] Using nproc_per_node=1.
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199] Starting elastic_operator with launch configs:
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   entrypoint         : nemo_run.core.runners.fdl_runner
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   min_nodes          : 1
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   max_nodes          : 1
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   nproc_per_node     : 1
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   run_id             : 4820
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   rdzv_backend       : c10d
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   rdzv_endpoint      : localhost:0
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   rdzv_configs       : {'timeout': 900}
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   max_restarts       : 0
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   monitor_interval   : 0.1
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   log_dir            : /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_full_4k_tokens_34940/qwen3_600m_full_4k_tokens_34940_1764256023/qwen3_600m_full_4k_tokens_34940/nemo_run/qwen3_600m_full_4k_tokens_34940-wrxhmhjfnd12dc/torchelastic/qwen3_600m_full_4k_tokens_34940
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   metrics_cfg        : {}
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199]   event_log_handler  : null
kens_34940/0 I1127 16:07:06.260000 2417549 torch/distributed/launcher/api.py:199] 
kens_34940/0 I1127 16:07:06.266000 2417549 torch/distributed/elastic/agent/server/api.py:869] [default] starting workers for entrypoint: python3
kens_34940/0 I1127 16:07:06.267000 2417549 torch/distributed/elastic/agent/server/api.py:677] [default] Rendezvous'ing worker group
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523] [default] Rendezvous complete for workers. Result:
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   restart_count=0
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   master_addr=hai003.haicore.berlin
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   master_port=36787
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   group_rank=0
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   group_world_size=1
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   local_ranks=[0]
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   role_ranks=[0]
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   global_ranks=[0]
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   role_world_sizes=[1]
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   global_world_sizes=[1]
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523]   event_log_handler=null
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:523] 
kens_34940/0 I1127 16:07:06.582000 2417549 torch/distributed/elastic/agent/server/api.py:685] [default] Starting worker group
kens_34940/0 I1127 16:07:06.583000 2417549 torch/distributed/elastic/agent/server/local_elastic_agent.py:296] use_agent_store: True
kens_34940/0 I1127 16:07:06.583000 2417549 torch/distributed/elastic/agent/server/local_elastic_agent.py:192] Environment variable 'TORCHELASTIC_ENABLE_FILE_TIMER' not found. Do not start FileTimerServer.
kens_34940/0 I1127 16:07:06.584000 2417549 torch/distributed/elastic/agent/server/local_elastic_agent.py:236] Environment variable 'TORCHELASTIC_HEALTH_CHECK_PORT' not found. Do not start health check.
kens_34940/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
kens_34940/0 [default0]:  import pynvml  # type: ignore[import]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 nemo_logger:162] Experiments will be logged at /fast/project/HFMI_SynergyUnit/mihir/logs/qwen3_600m_full_4k_tokens_34940
kens_34940/0 [default0]:GPU available: True (cuda), used: True
kens_34940/0 [default0]:TPU available: False, using: 0 TPU cores
kens_34940/0 [default0]:HPU available: False, using: 0 HPUs
kens_34940/0 [default0]:[NeMo W 2025-11-27 16:07:27 nemo_logger:139] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
kens_34940/0 [default0]:[NeMo W 2025-11-27 16:07:27 nemo_logger:190] "update_logger_directory" is True. Overwriting tensorboard logger "save_dir" to /fast/project/HFMI_SynergyUnit/mihir/logs/tb_logs
kens_34940/0 [default0]:[NeMo W 2025-11-27 16:07:27 nemo_logger:198] "update_logger_directory" is True. Overwriting wandb logger "save_dir" to /fast/project/HFMI_SynergyUnit/mihir/logs/wandb
kens_34940/0 [default0]:[NeMo W 2025-11-27 16:07:27 nemo_logger:209] The Trainer already contains a ModelCheckpoint callback. This will be overwritten.
kens_34940/0 [default0]:[NeMo W 2025-11-27 16:07:27 nemo_logger:239] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 2000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
kens_34940/0 [default0]:[NeMo W 2025-11-27 16:07:27 resume:47] Encountered error while trying to restore tokenizer. Tokenizer is not restored. Original error: Unable to instantiate HuggingFace AUTOTOKENIZER for /fast/project/HFMI_SynergyUnit/mihir/logs/qwen3_600m_full_4k_tokens_34940/checkpoints/model_name=0--val_loss=0.26-step=299-consumed_samples=9600.0-last/context/nemo_tokenizer. Exception: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/fast/project/HFMI_SynergyUnit/mihir/logs/qwen3_600m_full_4k_tokens_34940/checkpoints/model_name=0--val_loss=0.26-step=299-consumed_samples=9600.0-last/context/nemo_tokenizer'. Use `repo_type` argument if needed.
kens_34940/0 [default0]:    
kens_34940/0 [default0]:    Fiddle context: failed to construct or call AutoTokenizer at <root> with positional arguments: (), keyword arguments: (pretrained_model_name='/fast/project/HFMI_SynergyUnit/mihir/logs/qwen3_600m_full_4k_tokens_34940/checkpoints/model_name=0--val_loss=0.26-step=299-consumed_samples=9600.0-last/context/nemo_tokenizer', trust_remote_code=True).
kens_34940/0 [default0]:You are using a CUDA device ('NVIDIA H100 80GB HBM3') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_strategy:460] Fixing mis-match between ddp-config & mcore-optimizer config
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:449] Rank 0 has data parallel group : [0]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:455] Rank 0 has combined group of data parallel and context parallel : [0]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:460] All data parallel group ranks with context parallel combined: [[0]]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:463] Ranks 0 has data parallel rank: 0
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:471] Rank 0 has context parallel group: [0]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:474] All context parallel group ranks: [[0]]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:475] Ranks 0 has context parallel rank: 0
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:482] Rank 0 has model parallel group: [0]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:483] All model parallel group ranks: [[0]]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:492] Rank 0 has tensor model parallel group: [0]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:496] All tensor model parallel group ranks: [[0]]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:497] Rank 0 has tensor model parallel rank: 0
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:532] Rank 0 has pipeline model parallel group: [0]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:544] Rank 0 has embedding group: [0]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:550] All pipeline model parallel group ranks: [[0]]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:551] Rank 0 has pipeline model parallel rank 0
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:552] All embedding group ranks: [[0]]
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:27 megatron_init:553] Rank 0 has embedding rank: 0
kens_34940/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
kens_34940/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
kens_34940/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
kens_34940/0 [default0]:[W1127 16:07:27.501461842 ProcessGroupNCCL.cpp:981] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
kens_34940/0 [default0]:----------------------------------------------------------------------------------------------------
kens_34940/0 [default0]:distributed_backend=nccl
kens_34940/0 [default0]:All distributed processes registered. Starting with 1 processes
kens_34940/0 [default0]:----------------------------------------------------------------------------------------------------
kens_34940/0 [default0]:
kens_34940/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
kens_34940/0 [default0]:wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
kens_34940/0 [default0]:wandb: setting up run isfyiv90
kens_34940/0 [default0]:wandb: Tracking run with wandb version 0.23.0
kens_34940/0 [default0]:wandb: Run data is saved locally in /fast/project/HFMI_SynergyUnit/mihir/logs/wandb/wandb/run-20251127_160730-isfyiv90
kens_34940/0 [default0]:wandb: Run `wandb offline` to turn off syncing.
kens_34940/0 [default0]:wandb: Syncing run qwen3_600m_full_4k_tokens_34940
kens_34940/0 [default0]:wandb: ⭐️ View project at https://wandb.ai/instant-uv/crowd-pilot-nemo
kens_34940/0 [default0]:wandb: 🚀 View run at https://wandb.ai/instant-uv/crowd-pilot-nemo/runs/isfyiv90
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:33 megatron_strategy:472] Copying Trainer's 'max_steps' (2000) to LR scheduler's 'max_steps'.
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:33 num_microbatches_calculator:228] setting number of microbatches to constant 8
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:33 megatron_parallel:584]  > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 596049920
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:33 utils:673] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_megatron_fsdp=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache=False, nccl_ub=False, fsdp_double_buffer=False, outer_dp_sharding_strategy='no_shard', disable_symmetric_registration=False, delay_wgrad_compute=False)
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:33 utils:694] Number of buckets for gradient all-reduce / reduce-scatter: 1
kens_34940/0 [default0]:    Params for bucket 1 (596049920 elements, 596049920 padded size):
kens_34940/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.11.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.3.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.0.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.4.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.18.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.9.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.27.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.10.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.3.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.20.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.17.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.8.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.4.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.embedding.word_embeddings.weight
kens_34940/0 [default0]:    	module.decoder.layers.26.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.26.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.27.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.19.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.18.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.9.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.2.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.25.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.16.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.7.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.weight
kens_34940/0 [default0]:LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
kens_34940/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.17.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.8.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.24.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.15.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.2.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.25.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.16.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.7.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.1.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.21.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.23.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.14.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.13.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.21.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.24.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.15.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.6.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.13.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.22.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.12.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.23.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.14.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.5.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.final_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.20.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.6.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.11.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.weight
kens_34940/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.1.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.0.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.22.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.12.self_attention.k_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.weight
kens_34940/0 [default0]:    	module.decoder.layers.5.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.layer_norm_weight
kens_34940/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.weight
kens_34940/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.19.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.weight
kens_34940/0 [default0]:    	module.decoder.layers.10.self_attention.q_layernorm.weight
kens_34940/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.weight
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:33 utils:673] Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=5e-06, min_lr=None, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='delayed', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.98, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather=False, overlap_param_gather_with_optimizer_step=False, use_megatron_fsdp=False, optimizer_cpu_offload=False, optimizer_offload_fraction=0.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=False, timers=None, config_logger_dir='')
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:33 pl:411] Loaded sharded_state_dict_metadata from checkpoint: {'distrib_optim_sharding_type': 'fully_sharded_model_space'}
kens_34940/0 [default0]:
kens_34940/0 [default0]:  | Name   | Type | Params | Mode 
kens_34940/0 [default0]:----------------------------------------
kens_34940/0 [default0]:0 | module | DDP  | 596 M  | train
kens_34940/0 [default0]:----------------------------------------
kens_34940/0 [default0]:596 M     Trainable params
kens_34940/0 [default0]:0         Non-trainable params
kens_34940/0 [default0]:596 M     Total params
kens_34940/0 [default0]:2,384.200 Total estimated model params size (MB)
kens_34940/0 [default0]:571       Modules in train mode
kens_34940/0 [default0]:0         Modules in eval mode
kens_34940/0 [default0]:Restoring states from the checkpoint path at /fast/project/HFMI_SynergyUnit/mihir/logs/qwen3_600m_full_4k_tokens_34940/checkpoints/model_name=0--val_loss=0.24-step=599-consumed_samples=19200.0-last/weights
kens_34940/0 [default0]:[NeMo W 2025-11-27 16:07:33 serialization:184] DEPRECATED: Passing 'checkpoint_dir' as a Path object in load_common_state_dict will no longer be supported in a future release. Please pass it as a string instead.
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:33 pl:275] Using <megatron.core.dist_checkpointing.strategies.fully_parallel.FullyParallelLoadStrategyWrapper object at 0x7f8e88503fd0> dist-ckpt load strategy.
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:44 pl:297] Global Checkpoint Load : Rank : 0 : Start time : 1764256053.295s : Time spent in load_checkpoint: 10.925s
kens_34940/0 [default0]:[NeMo I 2025-11-27 16:07:44 distrib_optimizer:840] Loading distributed optimizer sharded state of type fully_sharded_model_space
kens_34940/0 [default0]:*** Loaded DataModule state dict successfully. IGNORE PTL's warning below about the dataloader not being resumable. This is warning is expected because we are handling dataloader resumption manually in NeMo. ***
kens_34940/0 [default0]:Restored all states from the checkpoint at /fast/project/HFMI_SynergyUnit/mihir/logs/qwen3_600m_full_4k_tokens_34940/checkpoints/model_name=0--val_loss=0.24-step=599-consumed_samples=19200.0-last/weights
kens_34940/0 [default0]:Traceback (most recent call last):
kens_34940/0 [default0]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main
kens_34940/0 [default0]:    return _run_code(code, main_globals, None,
kens_34940/0 [default0]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code
kens_34940/0 [default0]:    exec(code, run_globals)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 72, in <module>
kens_34940/0 [default0]:    fdl_runner_app()
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 327, in __call__
kens_34940/0 [default0]:    raise e
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 310, in __call__
kens_34940/0 [default0]:    return get_command(self)(*args, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1485, in __call__
kens_34940/0 [default0]:    return self.main(*args, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 719, in main
kens_34940/0 [default0]:    return _main(
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 192, in _main
kens_34940/0 [default0]:    rv = self.invoke(ctx)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1269, in invoke
kens_34940/0 [default0]:    return ctx.invoke(self.callback, **ctx.params)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 824, in invoke
kens_34940/0 [default0]:    return callback(*args, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 691, in wrapper
kens_34940/0 [default0]:    return callback(**use_params)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 68, in fdl_direct_run
kens_34940/0 [default0]:    fdl_fn()
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/api.py", line 237, in finetune
kens_34940/0 [default0]:    return train(
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/api.py", line 136, in train
kens_34940/0 [default0]:    trainer.fit(model, data)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 538, in fit
kens_34940/0 [default0]:    call._call_and_handle_interrupt(
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 46, in _call_and_handle_interrupt
kens_34940/0 [default0]:    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
kens_34940/0 [default0]:    return function(*args, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 574, in _fit_impl
kens_34940/0 [default0]:    self._run(model, ckpt_path=ckpt_path)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 981, in _run
kens_34940/0 [default0]:    results = self._run_stage()
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1025, in _run_stage
kens_34940/0 [default0]:    self.fit_loop.run()
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 197, in run
kens_34940/0 [default0]:    self.setup_data()
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 225, in setup_data
kens_34940/0 [default0]:    train_dataloader = _request_dataloader(source)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py", line 325, in _request_dataloader
kens_34940/0 [default0]:    return data_source.dataloader()
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py", line 292, in dataloader
kens_34940/0 [default0]:    return call._call_lightning_datamodule_hook(self.instance.trainer, self.name)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 189, in _call_lightning_datamodule_hook
kens_34940/0 [default0]:    return fn(*args, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/fine_tuning.py", line 207, in train_dataloader
kens_34940/0 [default0]:    self._create_dataset(
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/chat.py", line 80, in _create_dataset
kens_34940/0 [default0]:    return create_sft_dataset(
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/core.py", line 118, in create_sft_dataset
kens_34940/0 [default0]:    return GPTSFTChatDataset(
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/core.py", line 972, in __init__
kens_34940/0 [default0]:    super().__init__(
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/core.py", line 258, in __init__
kens_34940/0 [default0]:    self._build_samples_mapping()
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/core.py", line 1024, in _build_samples_mapping
kens_34940/0 [default0]:    id1 = self.tokenizer.text_to_ids(PREFIX_STR)
kens_34940/0 [default0]:AttributeError: 'NoneType' object has no attribute 'text_to_ids'
kens_34940/0 [default0]:[rank0]: Traceback (most recent call last):
kens_34940/0 [default0]:[rank0]:   File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main
kens_34940/0 [default0]:[rank0]:     return _run_code(code, main_globals, None,
kens_34940/0 [default0]:[rank0]:   File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code
kens_34940/0 [default0]:[rank0]:     exec(code, run_globals)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 72, in <module>
kens_34940/0 [default0]:[rank0]:     fdl_runner_app()
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 327, in __call__
kens_34940/0 [default0]:[rank0]:     raise e
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 310, in __call__
kens_34940/0 [default0]:[rank0]:     return get_command(self)(*args, **kwargs)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1485, in __call__
kens_34940/0 [default0]:[rank0]:     return self.main(*args, **kwargs)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 719, in main
kens_34940/0 [default0]:[rank0]:     return _main(
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 192, in _main
kens_34940/0 [default0]:[rank0]:     rv = self.invoke(ctx)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1269, in invoke
kens_34940/0 [default0]:[rank0]:     return ctx.invoke(self.callback, **ctx.params)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 824, in invoke
kens_34940/0 [default0]:[rank0]:     return callback(*args, **kwargs)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 691, in wrapper
kens_34940/0 [default0]:[rank0]:     return callback(**use_params)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 68, in fdl_direct_run
kens_34940/0 [default0]:[rank0]:     fdl_fn()
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/api.py", line 237, in finetune
kens_34940/0 [default0]:[rank0]:     return train(
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/api.py", line 136, in train
kens_34940/0 [default0]:[rank0]:     trainer.fit(model, data)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 538, in fit
kens_34940/0 [default0]:[rank0]:     call._call_and_handle_interrupt(
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 46, in _call_and_handle_interrupt
kens_34940/0 [default0]:[rank0]:     return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
kens_34940/0 [default0]:[rank0]:     return function(*args, **kwargs)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 574, in _fit_impl
kens_34940/0 [default0]:[rank0]:     self._run(model, ckpt_path=ckpt_path)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 981, in _run
kens_34940/0 [default0]:[rank0]:     results = self._run_stage()
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1025, in _run_stage
kens_34940/0 [default0]:[rank0]:     self.fit_loop.run()
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 197, in run
kens_34940/0 [default0]:[rank0]:     self.setup_data()
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 225, in setup_data
kens_34940/0 [default0]:[rank0]:     train_dataloader = _request_dataloader(source)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py", line 325, in _request_dataloader
kens_34940/0 [default0]:[rank0]:     return data_source.dataloader()
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py", line 292, in dataloader
kens_34940/0 [default0]:[rank0]:     return call._call_lightning_datamodule_hook(self.instance.trainer, self.name)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 189, in _call_lightning_datamodule_hook
kens_34940/0 [default0]:[rank0]:     return fn(*args, **kwargs)
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/fine_tuning.py", line 207, in train_dataloader
kens_34940/0 [default0]:[rank0]:     self._create_dataset(
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/chat.py", line 80, in _create_dataset
kens_34940/0 [default0]:[rank0]:     return create_sft_dataset(
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/core.py", line 118, in create_sft_dataset
kens_34940/0 [default0]:[rank0]:     return GPTSFTChatDataset(
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/core.py", line 972, in __init__
kens_34940/0 [default0]:[rank0]:     super().__init__(
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/core.py", line 258, in __init__
kens_34940/0 [default0]:[rank0]:     self._build_samples_mapping()
kens_34940/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/data/core.py", line 1024, in _build_samples_mapping
kens_34940/0 [default0]:[rank0]:     id1 = self.tokenizer.text_to_ids(PREFIX_STR)
kens_34940/0 [default0]:[rank0]: AttributeError: 'NoneType' object has no attribute 'text_to_ids'
kens_34940/0 [default0]:wandb: 
kens_34940/0 [default0]:wandb: 🚀 View run qwen3_600m_full_4k_tokens_34940 at: https://wandb.ai/instant-uv/crowd-pilot-nemo/runs/isfyiv90
kens_34940/0 [default0]:wandb: Find logs at: ../../../../fast/project/HFMI_SynergyUnit/mihir/logs/wandb/wandb/run-20251127_160730-isfyiv90/logs
kens_34940/0 E1127 16:07:51.571000 2417549 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 0 (pid: 2417565) of binary: /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/bin/python3
kens_34940/0 I1127 16:07:51.576000 2417549 torch/distributed/elastic/multiprocessing/errors/__init__.py:370] ('local_rank %s FAILED with no error file. Decorate your entrypoint fn with @record for traceback info. See: https://pytorch.org/docs/stable/elastic/errors.html', 0)
kens_34940/0 Traceback (most recent call last):
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/bin/torchrun", line 10, in <module>
kens_34940/0     sys.exit(main())
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
kens_34940/0     return f(*args, **kwargs)
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main
kens_34940/0     run(args)
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run
kens_34940/0     elastic_launch(
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__
kens_34940/0     return launch_agent(self._config, self._entrypoint, list(args))
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent
kens_34940/0     raise ChildFailedError(
kens_34940/0 torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
kens_34940/0 ============================================================
kens_34940/0 nemo_run.core.runners.fdl_runner FAILED
kens_34940/0 ------------------------------------------------------------
kens_34940/0 Failures:
kens_34940/0   <NO_OTHER_FAILURES>
kens_34940/0 ------------------------------------------------------------
kens_34940/0 Root Cause (first observed failure):
kens_34940/0 [0]:
kens_34940/0   time      : 2025-11-27_16:07:51
kens_34940/0   host      : hai003.haicore.berlin
kens_34940/0   rank      : 0 (local_rank: 0)
kens_34940/0   exitcode  : 1 (pid: 2417565)
kens_34940/0   error_file: <N/A>
kens_34940/0   traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
kens_34940/0 ============================================================
[16:07:53] INFO     Job                                          launcher.py:160
                    qwen3_600m_full_4k_tokens_34940-wrxhmhjfnd12                
                    dc finished: FAILED                                         
                                                                                
# The experiment was run with the following tasks: ['qwen3_600m_full_4k_tokens_3
# You can inspect and reconstruct this experiment at a later point in time using
experiment = run.Experiment.from_id("qwen3_600m_full_4k_tokens_34940_1764256023"
experiment.status() # Gets the overall status                                   
experiment.logs("qwen3_600m_full_4k_tokens_34940") # Gets the log for the provid
experiment.cancel("qwen3_600m_full_4k_tokens_34940") # Cancels the provided task
                                                                                
                                                                                
# You can inspect this experiment at a later point in time using the CLI as well
nemo experiment status qwen3_600m_full_4k_tokens_34940_1764256023               
nemo experiment logs qwen3_600m_full_4k_tokens_34940_1764256023 0               
nemo experiment cancel qwen3_600m_full_4k_tokens_34940_1764256023 0