[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/nemo/qwen3_600m_finetune_35073.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=00:15:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

python "$1"/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
─ Entering Experiment qwen3_600m_full_4k_tokens_35073 with id: qwen3_600m_ful… ─
[12:38:15] INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_full_4k_tokens_35073/                       
                    qwen3_600m_full_4k_tokens_35073_17643                       
                    29895/qwen3_600m_full_4k_tokens_35073                       
[12:38:15] Launching job qwen3_600m_full_4k_tokens_35073 for   experiment.py:744
           experiment qwen3_600m_full_4k_tokens_35073                           
           INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_full_4k_tokens_35073/                       
                    qwen3_600m_full_4k_tokens_35073_17643                       
                    29895/qwen3_600m_full_4k_tokens_35073                       
           INFO     Launched app:                                launcher.py:111
                    local_persistent://nemo_run/qwen3_600m_full_                
                    4k_tokens_35073-tfmw4pgk9bz6pc                              
─ Waiting for Experiment qwen3_600m_full_4k_tokens_35073_1764329895 to finish ──

Experiment Status for qwen3_600m_full_4k_tokens_35073_1764329895

Task 0: qwen3_600m_full_4k_tokens_35073
- Status: RUNNING
- Executor: LocalExecutor
- Job id: qwen3_600m_full_4k_tokens_35073-tfmw4pgk9bz6pc
- Local Directory: /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_full_4k_tokens_35073/qwen3_600m_full_4k_tokens_35073_1764329895/qwen3_600m_full_4k_tokens_35073

           INFO     Waiting for job                              launcher.py:130
                    qwen3_600m_full_4k_tokens_35073-tfmw4pgk9bz6                
                    pc to finish [log=True]...                                  
kens_35073/0 /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
kens_35073/0   import pynvml  # type: ignore[import]
kens_35073/0 I1128 12:38:17.853000 3415478 torch/distributed/run.py:657] Using nproc_per_node=1.
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199] Starting elastic_operator with launch configs:
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   entrypoint         : nemo_run.core.runners.fdl_runner
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   min_nodes          : 1
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   max_nodes          : 1
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   nproc_per_node     : 1
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   run_id             : 9065
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   rdzv_backend       : c10d
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   rdzv_endpoint      : localhost:0
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   rdzv_configs       : {'timeout': 900}
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   max_restarts       : 0
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   monitor_interval   : 0.1
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   log_dir            : /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_full_4k_tokens_35073/qwen3_600m_full_4k_tokens_35073_1764329895/qwen3_600m_full_4k_tokens_35073/nemo_run/qwen3_600m_full_4k_tokens_35073-tfmw4pgk9bz6pc/torchelastic/qwen3_600m_full_4k_tokens_35073
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   metrics_cfg        : {}
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199]   event_log_handler  : null
kens_35073/0 I1128 12:38:17.857000 3415478 torch/distributed/launcher/api.py:199] 
kens_35073/0 I1128 12:38:17.864000 3415478 torch/distributed/elastic/agent/server/api.py:869] [default] starting workers for entrypoint: python3
kens_35073/0 I1128 12:38:17.864000 3415478 torch/distributed/elastic/agent/server/api.py:677] [default] Rendezvous'ing worker group
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523] [default] Rendezvous complete for workers. Result:
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   restart_count=0
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   master_addr=hai008.haicore.berlin
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   master_port=38231
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   group_rank=0
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   group_world_size=1
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   local_ranks=[0]
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   role_ranks=[0]
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   global_ranks=[0]
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   role_world_sizes=[1]
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   global_world_sizes=[1]
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523]   event_log_handler=null
kens_35073/0 I1128 12:38:18.125000 3415478 torch/distributed/elastic/agent/server/api.py:523] 
kens_35073/0 I1128 12:38:18.126000 3415478 torch/distributed/elastic/agent/server/api.py:685] [default] Starting worker group
kens_35073/0 I1128 12:38:18.126000 3415478 torch/distributed/elastic/agent/server/local_elastic_agent.py:296] use_agent_store: True
kens_35073/0 I1128 12:38:18.127000 3415478 torch/distributed/elastic/agent/server/local_elastic_agent.py:192] Environment variable 'TORCHELASTIC_ENABLE_FILE_TIMER' not found. Do not start FileTimerServer.
kens_35073/0 I1128 12:38:18.128000 3415478 torch/distributed/elastic/agent/server/local_elastic_agent.py:236] Environment variable 'TORCHELASTIC_HEALTH_CHECK_PORT' not found. Do not start health check.
kens_35073/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
kens_35073/0 [default0]:  import pynvml  # type: ignore[import]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:40 nemo_logger:162] Experiments will be logged at /fast/project/HFMI_SynergyUnit/mihir/logs/qwen3_600m_full_4k_tokens_35073
kens_35073/0 [default0]:GPU available: True (cuda), used: True
kens_35073/0 [default0]:TPU available: False, using: 0 TPU cores
kens_35073/0 [default0]:HPU available: False, using: 0 HPUs
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:40 nemo_logger:139] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:40 nemo_logger:190] "update_logger_directory" is True. Overwriting tensorboard logger "save_dir" to /fast/project/HFMI_SynergyUnit/mihir/logs/tb_logs
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:40 nemo_logger:198] "update_logger_directory" is True. Overwriting wandb logger "save_dir" to /fast/project/HFMI_SynergyUnit/mihir/logs/wandb
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:40 nemo_logger:209] The Trainer already contains a ModelCheckpoint callback. This will be overwritten.
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:40 nemo_logger:239] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 2000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:40 megatron_strategy:460] Fixing mis-match between ddp-config & mcore-optimizer config
kens_35073/0 [default0]:You are using a CUDA device ('NVIDIA H100 80GB HBM3') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:449] Rank 0 has data parallel group : [0]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:455] Rank 0 has combined group of data parallel and context parallel : [0]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:460] All data parallel group ranks with context parallel combined: [[0]]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:463] Ranks 0 has data parallel rank: 0
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:471] Rank 0 has context parallel group: [0]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:474] All context parallel group ranks: [[0]]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:475] Ranks 0 has context parallel rank: 0
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:482] Rank 0 has model parallel group: [0]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:483] All model parallel group ranks: [[0]]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:492] Rank 0 has tensor model parallel group: [0]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:496] All tensor model parallel group ranks: [[0]]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:497] Rank 0 has tensor model parallel rank: 0
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:532] Rank 0 has pipeline model parallel group: [0]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:544] Rank 0 has embedding group: [0]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:550] All pipeline model parallel group ranks: [[0]]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:551] Rank 0 has pipeline model parallel rank 0
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:552] All embedding group ranks: [[0]]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:41 megatron_init:553] Rank 0 has embedding rank: 0
kens_35073/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
kens_35073/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
kens_35073/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
kens_35073/0 [default0]:[W1128 12:38:41.838669094 ProcessGroupNCCL.cpp:981] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
kens_35073/0 [default0]:----------------------------------------------------------------------------------------------------
kens_35073/0 [default0]:distributed_backend=nccl
kens_35073/0 [default0]:All distributed processes registered. Starting with 1 processes
kens_35073/0 [default0]:----------------------------------------------------------------------------------------------------
kens_35073/0 [default0]:
kens_35073/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
kens_35073/0 [default0]:wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
kens_35073/0 [default0]:wandb: setting up run 35073
kens_35073/0 [default0]:wandb: Tracking run with wandb version 0.23.0
kens_35073/0 [default0]:wandb: Run data is saved locally in /fast/project/HFMI_SynergyUnit/mihir/logs/wandb/wandb/run-20251128_123843-35073
kens_35073/0 [default0]:wandb: Run `wandb offline` to turn off syncing.
kens_35073/0 [default0]:wandb: Syncing run qwen3_600m_full_4k_tokens_35073
kens_35073/0 [default0]:wandb: ⭐️ View project at https://wandb.ai/instant-uv/crowd-pilot-nemo
kens_35073/0 [default0]:wandb: 🚀 View run at https://wandb.ai/instant-uv/crowd-pilot-nemo/runs/35073
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:45 base:373] Use preset vocab_size: 151936, original vocab_size: 151669, dummy tokens: 267.
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:45 megatron_strategy:472] Copying Trainer's 'max_steps' (2000) to LR scheduler's 'max_steps'.
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:45 num_microbatches_calculator:228] setting number of microbatches to constant 8
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:45 megatron_parallel:584]  > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 596049920
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:45 utils:673] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_megatron_fsdp=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache=False, nccl_ub=False, fsdp_double_buffer=False, outer_dp_sharding_strategy='no_shard', disable_symmetric_registration=False, delay_wgrad_compute=False)
kens_35073/0 [default0]:LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:45 utils:694] Number of buckets for gradient all-reduce / reduce-scatter: 1
kens_35073/0 [default0]:    Params for bucket 1 (596049920 elements, 596049920 padded size):
kens_35073/0 [default0]:    	module.decoder.layers.24.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.14.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.12.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.11.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.11.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.final_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.25.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.23.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.17.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.13.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.3.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.3.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.22.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.18.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.24.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.16.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.12.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.2.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.21.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.17.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.9.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.2.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.16.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.8.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.21.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.19.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.weight
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:45 serialization:184] DEPRECATED: Passing 'checkpoint_dir' as a Path object in load_common_state_dict will no longer be supported in a future release. Please pass it as a string instead.
kens_35073/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.7.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.1.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.1.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.15.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.7.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.20.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.8.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.15.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.0.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.27.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.6.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.20.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.19.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.10.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.9.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.26.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.5.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.4.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.27.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.23.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.6.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.25.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.weight
kens_35073/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.13.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.14.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.26.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.weight
kens_35073/0 [default0]:    	module.decoder.layers.18.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.weight
kens_35073/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.5.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.4.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.embedding.word_embeddings.weight
kens_35073/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.22.self_attention.k_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.10.self_attention.q_layernorm.weight
kens_35073/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.weight
kens_35073/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.layer_norm_weight
kens_35073/0 [default0]:    	module.decoder.layers.0.self_attention.k_layernorm.weight
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:45 utils:673] Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=5e-06, min_lr=None, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='delayed', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.98, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather=False, overlap_param_gather_with_optimizer_step=False, use_megatron_fsdp=False, optimizer_cpu_offload=False, optimizer_offload_fraction=0.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=False, timers=None, config_logger_dir='')
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:45 megatron_strategy:1045] Doing selective restore from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:45 pl:411] Loaded sharded_state_dict_metadata from checkpoint: {'distrib_optim_sharding_type': 'fully_sharded_model_space'}
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:45 pl:275] Using <megatron.core.dist_checkpointing.strategies.fully_parallel.FullyParallelLoadStrategyWrapper object at 0x7fb7a4676110> dist-ckpt load strategy.
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:46 pl:297] Global Checkpoint Load : Rank : 0 : Start time : 1764329925.972s : Time spent in load_checkpoint: 0.631s
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:46 megatron_strategy:1050] Restoring model weights from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:38:46 megatron_strategy:1058] Finished restoring from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True), cleaning up.
kens_35073/0 [default0]:
kens_35073/0 [default0]:  | Name   | Type | Params | Mode 
kens_35073/0 [default0]:----------------------------------------
kens_35073/0 [default0]:0 | module | DDP  | 596 M  | train
kens_35073/0 [default0]:----------------------------------------
kens_35073/0 [default0]:596 M     Trainable params
kens_35073/0 [default0]:0         Non-trainable params
kens_35073/0 [default0]:596 M     Total params
kens_35073/0 [default0]:2,384.200 Total estimated model params size (MB)
kens_35073/0 [default0]:571       Modules in train mode
kens_35073/0 [default0]:0         Modules in eval mode
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:46 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=9` in the `DataLoader` to improve performance.
kens_35073/0 [default0]:    
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:52 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('global_batch_size', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
kens_35073/0 [default0]:    
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:52 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('val_loss', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
kens_35073/0 [default0]:    
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:52 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=9` in the `DataLoader` to improve performance.
kens_35073/0 [default0]:    
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:57 rerun_state_machine:1300] Implicit initialization of Rerun State Machine!
kens_35073/0 [default0]:[NeMo W 2025-11-28 12:38:57 rerun_state_machine:238] RerunStateMachine initialized in mode disabled
kens_35073/0 [default0]:Training epoch 0, iteration 0/1999 | lr: 9.804e-08 | global_batch_size: 32 | global_step: 0 | reduced_train_loss: 1.072 | train_step_timing in s: 4.798
kens_35073/0 [default0]:Training epoch 0, iteration 1/1999 | lr: 1.961e-07 | global_batch_size: 32 | global_step: 1 | reduced_train_loss: 1.117 | train_step_timing in s: 3.63 | consumed_samples: 64
kens_35073/0 [default0]:Training epoch 0, iteration 2/1999 | lr: 2.941e-07 | global_batch_size: 32 | global_step: 2 | reduced_train_loss: 1.15 | train_step_timing in s: 3.438 | consumed_samples: 96
kens_35073/0 [default0]:Training epoch 0, iteration 3/1999 | lr: 3.922e-07 | global_batch_size: 32 | global_step: 3 | reduced_train_loss: 1.109 | train_step_timing in s: 3.487 | consumed_samples: 128
kens_35073/0 [default0]:Training epoch 0, iteration 4/1999 | lr: 4.902e-07 | global_batch_size: 32 | global_step: 4 | reduced_train_loss: 1.147 | train_step_timing in s: 3.423 | consumed_samples: 160
kens_35073/0 [default0]:Training epoch 0, iteration 5/1999 | lr: 5.882e-07 | global_batch_size: 32 | global_step: 5 | reduced_train_loss: 1.195 | train_step_timing in s: 3.457 | consumed_samples: 192
kens_35073/0 [default0]:Training epoch 0, iteration 6/1999 | lr: 6.863e-07 | global_batch_size: 32 | global_step: 6 | reduced_train_loss: 1.121 | train_step_timing in s: 3.459 | consumed_samples: 224
kens_35073/0 [default0]:Training epoch 0, iteration 7/1999 | lr: 7.843e-07 | global_batch_size: 32 | global_step: 7 | reduced_train_loss: 1.176 | train_step_timing in s: 3.412 | consumed_samples: 256
kens_35073/0 [default0]:Training epoch 0, iteration 8/1999 | lr: 8.824e-07 | global_batch_size: 32 | global_step: 8 | reduced_train_loss: 1.097 | train_step_timing in s: 3.448 | consumed_samples: 288
kens_35073/0 [default0]:Training epoch 0, iteration 9/1999 | lr: 9.804e-07 | global_batch_size: 32 | global_step: 9 | reduced_train_loss: 1.113 | train_step_timing in s: 3.382 | consumed_samples: 320
kens_35073/0 [default0]:Training epoch 0, iteration 10/1999 | lr: 1.078e-06 | global_batch_size: 32 | global_step: 10 | reduced_train_loss: 1.08 | train_step_timing in s: 3.395 | consumed_samples: 352
kens_35073/0 [default0]:Training epoch 0, iteration 11/1999 | lr: 1.176e-06 | global_batch_size: 32 | global_step: 11 | reduced_train_loss: 1.103 | train_step_timing in s: 3.44 | consumed_samples: 384
kens_35073/0 [default0]:Training epoch 0, iteration 12/1999 | lr: 1.275e-06 | global_batch_size: 32 | global_step: 12 | reduced_train_loss: 1.231 | train_step_timing in s: 3.408 | consumed_samples: 416
kens_35073/0 [default0]:Training epoch 0, iteration 13/1999 | lr: 1.373e-06 | global_batch_size: 32 | global_step: 13 | reduced_train_loss: 1.035 | train_step_timing in s: 3.397 | consumed_samples: 448
kens_35073/0 [default0]:Training epoch 0, iteration 14/1999 | lr: 1.471e-06 | global_batch_size: 32 | global_step: 14 | reduced_train_loss: 0.8517 | train_step_timing in s: 3.829 | consumed_samples: 480
kens_35073/0 [default0]:Training epoch 0, iteration 15/1999 | lr: 1.569e-06 | global_batch_size: 32 | global_step: 15 | reduced_train_loss: 0.9389 | train_step_timing in s: 3.397 | consumed_samples: 512
kens_35073/0 [default0]:Training epoch 0, iteration 16/1999 | lr: 1.667e-06 | global_batch_size: 32 | global_step: 16 | reduced_train_loss: 0.946 | train_step_timing in s: 3.424 | consumed_samples: 544
kens_35073/0 [default0]:Training epoch 0, iteration 17/1999 | lr: 1.765e-06 | global_batch_size: 32 | global_step: 17 | reduced_train_loss: 1.131 | train_step_timing in s: 3.371 | consumed_samples: 576
kens_35073/0 [default0]:Training epoch 0, iteration 18/1999 | lr: 1.863e-06 | global_batch_size: 32 | global_step: 18 | reduced_train_loss: 1.085 | train_step_timing in s: 3.429 | consumed_samples: 608
kens_35073/0 [default0]:Training epoch 0, iteration 19/1999 | lr: 1.961e-06 | global_batch_size: 32 | global_step: 19 | reduced_train_loss: 0.8572 | train_step_timing in s: 3.403 | consumed_samples: 640
kens_35073/0 [default0]:Training epoch 0, iteration 20/1999 | lr: 2.059e-06 | global_batch_size: 32 | global_step: 20 | reduced_train_loss: 0.8213 | train_step_timing in s: 3.427 | consumed_samples: 672
kens_35073/0 [default0]:Training epoch 0, iteration 21/1999 | lr: 2.157e-06 | global_batch_size: 32 | global_step: 21 | reduced_train_loss: 1.002 | train_step_timing in s: 3.447 | consumed_samples: 704
kens_35073/0 [default0]:Training epoch 0, iteration 22/1999 | lr: 2.255e-06 | global_batch_size: 32 | global_step: 22 | reduced_train_loss: 0.692 | train_step_timing in s: 3.463 | consumed_samples: 736
kens_35073/0 [default0]:Training epoch 0, iteration 23/1999 | lr: 2.353e-06 | global_batch_size: 32 | global_step: 23 | reduced_train_loss: 0.9658 | train_step_timing in s: 3.468 | consumed_samples: 768
kens_35073/0 [default0]:Training epoch 0, iteration 24/1999 | lr: 2.451e-06 | global_batch_size: 32 | global_step: 24 | reduced_train_loss: 0.7248 | train_step_timing in s: 3.426 | consumed_samples: 800
kens_35073/0 [default0]:Training epoch 0, iteration 25/1999 | lr: 2.549e-06 | global_batch_size: 32 | global_step: 25 | reduced_train_loss: 0.6566 | train_step_timing in s: 3.447 | consumed_samples: 832
kens_35073/0 [default0]:Training epoch 0, iteration 26/1999 | lr: 2.647e-06 | global_batch_size: 32 | global_step: 26 | reduced_train_loss: 0.8284 | train_step_timing in s: 3.395 | consumed_samples: 864
kens_35073/0 [default0]:Training epoch 0, iteration 27/1999 | lr: 2.745e-06 | global_batch_size: 32 | global_step: 27 | reduced_train_loss: 0.6893 | train_step_timing in s: 3.505 | consumed_samples: 896
kens_35073/0 [default0]:Training epoch 0, iteration 28/1999 | lr: 2.843e-06 | global_batch_size: 32 | global_step: 28 | reduced_train_loss: 0.7295 | train_step_timing in s: 3.445 | consumed_samples: 928
kens_35073/0 [default0]:Training epoch 0, iteration 29/1999 | lr: 2.941e-06 | global_batch_size: 32 | global_step: 29 | reduced_train_loss: 0.8899 | train_step_timing in s: 3.421 | consumed_samples: 960
kens_35073/0 [default0]:Training epoch 0, iteration 30/1999 | lr: 3.039e-06 | global_batch_size: 32 | global_step: 30 | reduced_train_loss: 0.9172 | train_step_timing in s: 3.47 | consumed_samples: 992
kens_35073/0 [default0]:Training epoch 0, iteration 31/1999 | lr: 3.137e-06 | global_batch_size: 32 | global_step: 31 | reduced_train_loss: 0.5857 | train_step_timing in s: 3.502 | consumed_samples: 1024
kens_35073/0 [default0]:Training epoch 0, iteration 32/1999 | lr: 3.235e-06 | global_batch_size: 32 | global_step: 32 | reduced_train_loss: 0.6109 | train_step_timing in s: 3.414 | consumed_samples: 1056
kens_35073/0 [default0]:Training epoch 0, iteration 33/1999 | lr: 3.333e-06 | global_batch_size: 32 | global_step: 33 | reduced_train_loss: 0.7251 | train_step_timing in s: 3.415 | consumed_samples: 1088
kens_35073/0 [default0]:Training epoch 0, iteration 34/1999 | lr: 3.431e-06 | global_batch_size: 32 | global_step: 34 | reduced_train_loss: 0.53 | train_step_timing in s: 3.44 | consumed_samples: 1120
kens_35073/0 [default0]:Training epoch 0, iteration 35/1999 | lr: 3.529e-06 | global_batch_size: 32 | global_step: 35 | reduced_train_loss: 0.683 | train_step_timing in s: 3.432 | consumed_samples: 1152
kens_35073/0 [default0]:Training epoch 0, iteration 36/1999 | lr: 3.627e-06 | global_batch_size: 32 | global_step: 36 | reduced_train_loss: 0.6992 | train_step_timing in s: 3.361 | consumed_samples: 1184
kens_35073/0 [default0]:Training epoch 0, iteration 37/1999 | lr: 3.725e-06 | global_batch_size: 32 | global_step: 37 | reduced_train_loss: 0.7847 | train_step_timing in s: 3.454 | consumed_samples: 1216
kens_35073/0 [default0]:Training epoch 0, iteration 38/1999 | lr: 3.824e-06 | global_batch_size: 32 | global_step: 38 | reduced_train_loss: 0.6547 | train_step_timing in s: 3.485 | consumed_samples: 1248
kens_35073/0 [default0]:Training epoch 0, iteration 39/1999 | lr: 3.922e-06 | global_batch_size: 32 | global_step: 39 | reduced_train_loss: 1.028 | train_step_timing in s: 3.382 | consumed_samples: 1280
kens_35073/0 [default0]:Training epoch 0, iteration 40/1999 | lr: 4.02e-06 | global_batch_size: 32 | global_step: 40 | reduced_train_loss: 0.6099 | train_step_timing in s: 3.416 | consumed_samples: 1312
kens_35073/0 [default0]:Training epoch 0, iteration 41/1999 | lr: 4.118e-06 | global_batch_size: 32 | global_step: 41 | reduced_train_loss: 0.6125 | train_step_timing in s: 3.392 | consumed_samples: 1344
kens_35073/0 [default0]:Training epoch 0, iteration 42/1999 | lr: 4.216e-06 | global_batch_size: 32 | global_step: 42 | reduced_train_loss: 0.6558 | train_step_timing in s: 3.422 | consumed_samples: 1376
kens_35073/0 [default0]:Training epoch 0, iteration 43/1999 | lr: 4.314e-06 | global_batch_size: 32 | global_step: 43 | reduced_train_loss: 0.6744 | train_step_timing in s: 3.408 | consumed_samples: 1408
kens_35073/0 [default0]:Training epoch 0, iteration 44/1999 | lr: 4.412e-06 | global_batch_size: 32 | global_step: 44 | reduced_train_loss: 0.4855 | train_step_timing in s: 3.443 | consumed_samples: 1440
kens_35073/0 [default0]:Training epoch 0, iteration 45/1999 | lr: 4.51e-06 | global_batch_size: 32 | global_step: 45 | reduced_train_loss: 0.462 | train_step_timing in s: 3.425 | consumed_samples: 1472
kens_35073/0 [default0]:Training epoch 0, iteration 46/1999 | lr: 4.608e-06 | global_batch_size: 32 | global_step: 46 | reduced_train_loss: 0.5074 | train_step_timing in s: 3.432 | consumed_samples: 1504
kens_35073/0 [default0]:Training epoch 0, iteration 47/1999 | lr: 4.706e-06 | global_batch_size: 32 | global_step: 47 | reduced_train_loss: 0.4579 | train_step_timing in s: 3.4 | consumed_samples: 1536
kens_35073/0 [default0]:Training epoch 0, iteration 48/1999 | lr: 4.804e-06 | global_batch_size: 32 | global_step: 48 | reduced_train_loss: 0.4944 | train_step_timing in s: 3.43 | consumed_samples: 1568
kens_35073/0 [default0]:Training epoch 0, iteration 49/1999 | lr: 4.902e-06 | global_batch_size: 32 | global_step: 49 | reduced_train_loss: 0.6621 | train_step_timing in s: 3.404 | consumed_samples: 1600
kens_35073/0 [default0]:Validation: iteration 1/0
kens_35073/0 [default0]:Validation: iteration 2/0
kens_35073/0 [default0]:Validation: iteration 3/0
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:42:35 pl:353] Using FullyParallelSaveStrategyWrapper(torch_dist, 1) dist-ckpt save strategy.
kens_35073/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
kens_35073/0 [default0]:  import pynvml  # type: ignore[import]
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:42:40 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 50 : Start time: 1764330155.602s : Save duration: 4.884s
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:42:44 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/mihir/logs/qwen3_600m_full_4k_tokens_35073/checkpoints/model_name=0--val_loss=0.52-step=49-consumed_samples=1600.0-last.ckpt
kens_35073/0 [default0]:Training epoch 0, iteration 50/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 50 | reduced_train_loss: 0.4989 | train_step_timing in s: 3.629 | consumed_samples: 1632 | val_loss: 0.5151
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:42:47 dist_ckpt_io:160] Async finalization time took 0.000 s
kens_35073/0 [default0]:Training epoch 0, iteration 51/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 51 | reduced_train_loss: 0.4117 | train_step_timing in s: 3.555 | consumed_samples: 1664 | val_loss: 0.5151
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:42:51 dist_ckpt_io:160] Async finalization time took 0.000 s
kens_35073/0 [default0]:Training epoch 0, iteration 52/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 52 | reduced_train_loss: 0.4469 | train_step_timing in s: 3.675 | consumed_samples: 1696 | val_loss: 0.5151
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:42:55 dist_ckpt_io:160] Async finalization time took 0.000 s
kens_35073/0 [default0]:Training epoch 0, iteration 53/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 53 | reduced_train_loss: 0.3931 | train_step_timing in s: 3.724 | consumed_samples: 1728 | val_loss: 0.5151
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:42:58 dist_ckpt_io:160] Async finalization time took 0.000 s
kens_35073/0 [default0]:Training epoch 0, iteration 54/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 54 | reduced_train_loss: 0.5981 | train_step_timing in s: 3.58 | consumed_samples: 1760 | val_loss: 0.5151
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:43:02 dist_ckpt_io:160] Async finalization time took 0.000 s
kens_35073/0 [default0]:Training epoch 0, iteration 55/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 55 | reduced_train_loss: 0.677 | train_step_timing in s: 3.61 | consumed_samples: 1792 | val_loss: 0.5151
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:43:06 dist_ckpt_io:160] Async finalization time took 0.000 s
kens_35073/0 [default0]:Training epoch 0, iteration 56/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 56 | reduced_train_loss: 0.508 | train_step_timing in s: 3.684 | consumed_samples: 1824 | val_loss: 0.5151
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:43:10 pl:222] Successfully saved checkpoint from iteration      50 to /fast/project/HFMI_SynergyUnit/mihir/logs/qwen3_600m_full_4k_tokens_35073/checkpoints/model_name=0--val_loss=0.52-step=49-consumed_samples=1600.0-last.ckpt
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:43:10 model_checkpoint:657] Async checkpoint save for step 50 (/fast/project/HFMI_SynergyUnit/mihir/logs/qwen3_600m_full_4k_tokens_35073/checkpoints/model_name=0--val_loss=0.52-step=49-consumed_samples=1600.0-last.ckpt) finalized successfully.
kens_35073/0 [default0]:[NeMo I 2025-11-28 12:43:10 dist_ckpt_io:160] Async finalization time took 0.477 s
kens_35073/0 [default0]:Training epoch 0, iteration 57/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 57 | reduced_train_loss: 0.4712 | train_step_timing in s: 3.488 | consumed_samples: 1856 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 58/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 58 | reduced_train_loss: 0.4936 | train_step_timing in s: 3.45 | consumed_samples: 1888 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 59/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 59 | reduced_train_loss: 0.5156 | train_step_timing in s: 3.446 | consumed_samples: 1920 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 60/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 60 | reduced_train_loss: 0.4372 | train_step_timing in s: 3.44 | consumed_samples: 1952 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 61/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 61 | reduced_train_loss: 0.5487 | train_step_timing in s: 3.409 | consumed_samples: 1984 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 62/1999 | lr: 5e-06 | global_batch_size: 32 | global_step: 62 | reduced_train_loss: 0.4822 | train_step_timing in s: 3.447 | consumed_samples: 2016 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 63/1999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 63 | reduced_train_loss: 0.4165 | train_step_timing in s: 3.435 | consumed_samples: 2048 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 64/1999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 64 | reduced_train_loss: 0.4523 | train_step_timing in s: 3.469 | consumed_samples: 2080 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 65/1999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 65 | reduced_train_loss: 0.4413 | train_step_timing in s: 3.438 | consumed_samples: 2112 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 66/1999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 66 | reduced_train_loss: 0.323 | train_step_timing in s: 3.453 | consumed_samples: 2144 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 67/1999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 67 | reduced_train_loss: 0.3876 | train_step_timing in s: 3.402 | consumed_samples: 2176 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 68/1999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 68 | reduced_train_loss: 0.3781 | train_step_timing in s: 3.435 | consumed_samples: 2208 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 69/1999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 69 | reduced_train_loss: 0.5771 | train_step_timing in s: 3.432 | consumed_samples: 2240 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 70/1999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 70 | reduced_train_loss: 0.4121 | train_step_timing in s: 3.415 | consumed_samples: 2272 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 71/1999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 71 | reduced_train_loss: 0.3887 | train_step_timing in s: 3.442 | consumed_samples: 2304 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 72/1999 | lr: 4.998e-06 | global_batch_size: 32 | global_step: 72 | reduced_train_loss: 0.3808 | train_step_timing in s: 3.425 | consumed_samples: 2336 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 73/1999 | lr: 4.998e-06 | global_batch_size: 32 | global_step: 73 | reduced_train_loss: 0.3956 | train_step_timing in s: 3.427 | consumed_samples: 2368 | val_loss: 0.5151
kens_35073/0 [default0]:Training epoch 0, iteration 74/1999 | lr: 4.998e-06 | global_batch_size: 32 | global_step: 74 | reduced_train_loss: 0.3707 | train_step_timing in s: 3.434 | consumed_samples: 2400 | val_loss: 0.5151
[2025-11-28T12:53:10.002] error: *** JOB 35073 ON hai008 CANCELLED AT 2025-11-28T12:53:10 DUE TO TIME LIMIT ***