[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/nemo/qwen3_600m_finetune_34694.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

python "$1"/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
─ Entering Experiment qwen3_600m_lora_finetune_crowd_code_dataset_32k with id… ─
[14:26:51] INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_lora_finetune_crowd_c                       
                    ode_dataset_32k/qwen3_600m_lora_finet                       
                    une_crowd_code_dataset_32k_1764077211                       
                    /qwen3_600m_lora_finetune_crowd_code_                       
                    dataset_32k                                                 
[14:26:51] Launching job                                       experiment.py:744
           qwen3_600m_lora_finetune_crowd_code_dataset_32k for                  
           experiment                                                           
           qwen3_600m_lora_finetune_crowd_code_dataset_32k                      
           INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_lora_finetune_crowd_c                       
                    ode_dataset_32k/qwen3_600m_lora_finet                       
                    une_crowd_code_dataset_32k_1764077211                       
                    /qwen3_600m_lora_finetune_crowd_code_                       
                    dataset_32k                                                 
           INFO     Launched app:                                launcher.py:111
                    local_persistent://nemo_run/qwen3_600m_lora_                
                    finetune_crowd_code_dataset_32k-wf4x4zvfwvtn                
                    wd                                                          
─ Waiting for Experiment qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764… ─

Experiment Status for qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764077211

Task 0: qwen3_600m_lora_finetune_crowd_code_dataset_32k
- Status: RUNNING
- Executor: LocalExecutor
- Job id: qwen3_600m_lora_finetune_crowd_code_dataset_32k-wf4x4zvfwvtnwd
- Local Directory: /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_lora_finetune_crowd_code_dataset_32k/qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764077211/qwen3_600m_lora_finetune_crowd_code_dataset_32k

           INFO     Waiting for job                              launcher.py:130
                    qwen3_600m_lora_finetune_crowd_code_dataset_                
                    32k-wf4x4zvfwvtnwd to finish [log=True]...                  
ataset_32k/0 /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0   import pynvml  # type: ignore[import]
ataset_32k/0 I1125 14:26:54.095000 754691 torch/distributed/run.py:657] Using nproc_per_node=2.
ataset_32k/0 W1125 14:26:54.096000 754691 torch/distributed/run.py:774] 
ataset_32k/0 W1125 14:26:54.096000 754691 torch/distributed/run.py:774] *****************************************
ataset_32k/0 W1125 14:26:54.096000 754691 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
ataset_32k/0 W1125 14:26:54.096000 754691 torch/distributed/run.py:774] *****************************************
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199] Starting elastic_operator with launch configs:
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   entrypoint         : nemo_run.core.runners.fdl_runner
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   min_nodes          : 1
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   max_nodes          : 1
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   nproc_per_node     : 2
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   run_id             : 8363
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   rdzv_backend       : c10d
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   rdzv_endpoint      : localhost:0
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   rdzv_configs       : {'timeout': 900}
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   max_restarts       : 0
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   monitor_interval   : 0.1
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   log_dir            : /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_lora_finetune_crowd_code_dataset_32k/qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764077211/qwen3_600m_lora_finetune_crowd_code_dataset_32k/nemo_run/qwen3_600m_lora_finetune_crowd_code_dataset_32k-wf4x4zvfwvtnwd/torchelastic/qwen3_600m_lora_finetune_crowd_code_dataset_32k
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   metrics_cfg        : {}
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199]   event_log_handler  : null
ataset_32k/0 I1125 14:26:54.099000 754691 torch/distributed/launcher/api.py:199] 
ataset_32k/0 I1125 14:26:54.105000 754691 torch/distributed/elastic/agent/server/api.py:869] [default] starting workers for entrypoint: python3
ataset_32k/0 I1125 14:26:54.106000 754691 torch/distributed/elastic/agent/server/api.py:677] [default] Rendezvous'ing worker group
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523] [default] Rendezvous complete for workers. Result:
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   restart_count=0
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   master_addr=hai004.haicore.berlin
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   master_port=46323
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   group_rank=0
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   group_world_size=1
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   local_ranks=[0, 1]
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   role_ranks=[0, 1]
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   global_ranks=[0, 1]
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   role_world_sizes=[2, 2]
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   global_world_sizes=[2, 2]
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523]   event_log_handler=null
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:523] 
ataset_32k/0 I1125 14:26:54.200000 754691 torch/distributed/elastic/agent/server/api.py:685] [default] Starting worker group
ataset_32k/0 I1125 14:26:54.201000 754691 torch/distributed/elastic/agent/server/local_elastic_agent.py:296] use_agent_store: True
ataset_32k/0 I1125 14:26:54.202000 754691 torch/distributed/elastic/agent/server/local_elastic_agent.py:192] Environment variable 'TORCHELASTIC_ENABLE_FILE_TIMER' not found. Do not start FileTimerServer.
ataset_32k/0 I1125 14:26:54.202000 754691 torch/distributed/elastic/agent/server/local_elastic_agent.py:236] Environment variable 'TORCHELASTIC_HEALTH_CHECK_PORT' not found. Do not start health check.
ataset_32k/0 [default1]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0 [default1]:  import pynvml  # type: ignore[import]
ataset_32k/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0 [default0]:  import pynvml  # type: ignore[import]
ataset_32k/0 [default1]:trainer.num_devices: 2
ataset_32k/0 [default1]:trainer.num_nodes: 1
ataset_32k/0 [default1]:trainer.strategy.tensor_model_parallel_size: 2
ataset_32k/0 [default1]:trainer.strategy.pipeline_model_parallel_size: 1
ataset_32k/0 [default1]:trainer.strategy.context_parallel_size: 1
ataset_32k/0 [default1]:data.global_batch_size: 4
ataset_32k/0 [default1]:data.micro_batch_size: 1
ataset_32k/0 [default1]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default1]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default1]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default1]:[W1125 14:27:16.115490741 ProcessGroupNCCL.cpp:981] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
ataset_32k/0 [default0]:trainer.num_devices: 2
ataset_32k/0 [default0]:trainer.num_nodes: 1
ataset_32k/0 [default0]:trainer.strategy.tensor_model_parallel_size: 2
ataset_32k/0 [default0]:trainer.strategy.pipeline_model_parallel_size: 1
ataset_32k/0 [default0]:trainer.strategy.context_parallel_size: 1
ataset_32k/0 [default0]:data.global_batch_size: 4
ataset_32k/0 [default0]:data.micro_batch_size: 1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 api:1251] Disabling try_restore_best_ckpt restoration for adapters
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 nemo_logger:162] Experiments will be logged at /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21
ataset_32k/0 [default0]:GPU available: True (cuda), used: True
ataset_32k/0 [default0]:TPU available: False, using: 0 TPU cores
ataset_32k/0 [default0]:HPU available: False, using: 0 HPUs
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:27:21 nemo_logger:190] "update_logger_directory" is True. Overwriting tensorboard logger "save_dir" to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/tb_logs
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:27:21 nemo_logger:209] The Trainer already contains a ModelCheckpoint callback. This will be overwritten.
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:27:21 nemo_logger:239] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_strategy:460] Fixing mis-match between ddp-config & mcore-optimizer config
ataset_32k/0 [default0]:You are using a CUDA device ('NVIDIA H100 80GB HBM3') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:449] Rank 0 has data parallel group : [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:455] Rank 0 has combined group of data parallel and context parallel : [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:460] All data parallel group ranks with context parallel combined: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:463] Ranks 0 has data parallel rank: 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:471] Rank 0 has context parallel group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:474] All context parallel group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:475] Ranks 0 has context parallel rank: 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:482] Rank 0 has model parallel group: [0, 1]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:483] All model parallel group ranks: [[0, 1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:492] Rank 0 has tensor model parallel group: [0, 1]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:496] All tensor model parallel group ranks: [[0, 1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:497] Rank 0 has tensor model parallel rank: 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:518] All expert tensor parallel group ranks: [[0, 1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:519] Rank 0 has expert tensor parallel rank: 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:532] Rank 0 has pipeline model parallel group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:544] Rank 0 has embedding group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:550] All pipeline model parallel group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:551] Rank 0 has pipeline model parallel rank 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:552] All embedding group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:21 megatron_init:553] Rank 0 has embedding rank: 0
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default0]:[W1125 14:27:21.324584552 ProcessGroupNCCL.cpp:981] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
ataset_32k/0 [default0]:----------------------------------------------------------------------------------------------------
ataset_32k/0 [default0]:distributed_backend=nccl
ataset_32k/0 [default0]:All distributed processes registered. Starting with 2 processes
ataset_32k/0 [default0]:----------------------------------------------------------------------------------------------------
ataset_32k/0 [default0]:
ataset_32k/0 [default1]:[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:22 model_transform:67] Setting up ModelTransform for stage: fit
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:22 model_transform:70] Found model_transform attribute on pl_module
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:22 model_transform:73] Set model_transform to: <function _call_counter.<locals>.wrapper at 0x7fc48601f7f0>
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:22 base:373] Use preset vocab_size: 151936, original vocab_size: 151669, dummy tokens: 267.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:22 megatron_strategy:472] Copying Trainer's 'max_steps' (1000) to LR scheduler's 'max_steps'.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:22 num_microbatches_calculator:228] setting number of microbatches to constant 4
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:22 megatron_strategy:1045] Doing selective restore from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:22 pl:411] Loaded sharded_state_dict_metadata from checkpoint: {'distrib_optim_sharding_type': 'fully_sharded_model_space'}
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:22 pl:275] Using <megatron.core.dist_checkpointing.strategies.fully_parallel.FullyParallelLoadStrategyWrapper object at 0x7fc4b5e623b0> dist-ckpt load strategy.
ataset_32k/0 [default1]:LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1]
ataset_32k/0 [default0]:LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:27:22 serialization:184] DEPRECATED: Passing 'checkpoint_dir' as a Path object in load_common_state_dict will no longer be supported in a future release. Please pass it as a string instead.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 pl:297] Global Checkpoint Load : Rank : 0 : Start time : 1764077242.678s : Time spent in load_checkpoint: 0.804s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 megatron_strategy:1050] Restoring model weights from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 megatron_strategy:1058] Finished restoring from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True), cleaning up.
ataset_32k/0 [default0]:
ataset_32k/0 [default0]:  | Name   | Type     | Params | Mode 
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:0 | module | GPTModel | 298 M  | train
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:298 M     Trainable params
ataset_32k/0 [default0]:0         Non-trainable params
ataset_32k/0 [default0]:298 M     Total params
ataset_32k/0 [default0]:1,192.231 Total estimated model params size (MB)
ataset_32k/0 [default0]:569       Modules in train mode
ataset_32k/0 [default0]:0         Modules in eval mode
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.0.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.0.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.0.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.0.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.1.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.1.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.1.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.1.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.2.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.2.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.2.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.2.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.3.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.3.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.3.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.3.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.4.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.4.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.4.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.4.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.5.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.5.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.5.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.5.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.6.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.6.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.6.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.6.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.7.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.7.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.7.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:27:23 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=4` in the `DataLoader` to improve performance.
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.7.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.8.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.8.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.8.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.8.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.9.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.9.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.9.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.9.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.10.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.10.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.10.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.10.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.11.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.11.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.11.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.11.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.12.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.12.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.12.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.12.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.13.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.13.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.13.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.13.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.14.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.14.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.14.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.14.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.15.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.15.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.15.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.15.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.16.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.16.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.16.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.16.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.17.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.17.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.17.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.17.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.18.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.18.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.18.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.18.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.19.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.19.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.19.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.19.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.20.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.20.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.20.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.20.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.21.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.21.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.21.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.21.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.22.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.22.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.22.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.22.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.23.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.23.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.23.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.23.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.24.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.24.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.24.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.24.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.25.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.25.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.25.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.25.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.26.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.26.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.26.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.26.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.27.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.27.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.27.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 lora:468] Adding lora to: module.decoder.layers.27.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 model_transform:97] After applying model_transform:
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:  | Name   | Type     | Params | Mode 
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:0 | module | GPTModel | 306 M  | train
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:8.7 M     Trainable params
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:298 M     Non-trainable params
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:306 M     Total params
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:1,227.096 Total estimated model params size (MB)
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:1129      Modules in train mode
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:0         Modules in eval mode
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 peft:250] Initializing model parallel
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 megatron_parallel:584]  > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 306774016
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:23 megatron_parallel:587]  > number of trainable parameters: 8716288 (2.84% of total)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:24 utils:673] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_megatron_fsdp=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache=False, nccl_ub=False, fsdp_double_buffer=False, outer_dp_sharding_strategy='no_shard', disable_symmetric_registration=False, delay_wgrad_compute=False)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:24 utils:694] Number of buckets for gradient all-reduce / reduce-scatter: 1
ataset_32k/0 [default0]:    Params for bucket 1 (8716288 elements, 8716288 padded size):
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:24 peft:254] Setting up optimizers
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:27:24 utils:673] Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=None, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='delayed', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.98, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather=False, overlap_param_gather_with_optimizer_step=False, use_megatron_fsdp=False, optimizer_cpu_offload=False, optimizer_offload_fraction=0.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=False, timers=None, config_logger_dir='')
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:27:55 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('global_batch_size', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:27:55 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('val_loss', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:27:55 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=4` in the `DataLoader` to improve performance.
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:28:09 rerun_state_machine:1300] Implicit initialization of Rerun State Machine!
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:28:09 rerun_state_machine:238] RerunStateMachine initialized in mode disabled
ataset_32k/0 [default0]:Training epoch 0, iteration 0/999 | lr: 1.961e-06 | global_batch_size: 4 | global_step: 0 | reduced_train_loss: 0.597 | train_step_timing in s: 14.31
ataset_32k/0 [default0]:Training epoch 0, iteration 1/999 | lr: 3.922e-06 | global_batch_size: 4 | global_step: 1 | reduced_train_loss: 0.6534 | train_step_timing in s: 14.92 | consumed_samples: 8
ataset_32k/0 [default0]:Training epoch 0, iteration 2/999 | lr: 5.882e-06 | global_batch_size: 4 | global_step: 2 | reduced_train_loss: 0.4499 | train_step_timing in s: 14.09 | consumed_samples: 12
ataset_32k/0 [default0]:Training epoch 0, iteration 3/999 | lr: 7.843e-06 | global_batch_size: 4 | global_step: 3 | reduced_train_loss: 0.4421 | train_step_timing in s: 13.74 | consumed_samples: 16
ataset_32k/0 [default0]:Training epoch 0, iteration 4/999 | lr: 9.804e-06 | global_batch_size: 4 | global_step: 4 | reduced_train_loss: 0.6394 | train_step_timing in s: 14.02 | consumed_samples: 20
ataset_32k/0 [default0]:Training epoch 0, iteration 5/999 | lr: 1.176e-05 | global_batch_size: 4 | global_step: 5 | reduced_train_loss: 0.4294 | train_step_timing in s: 13.86 | consumed_samples: 24
ataset_32k/0 [default0]:Training epoch 0, iteration 6/999 | lr: 1.373e-05 | global_batch_size: 4 | global_step: 6 | reduced_train_loss: 0.545 | train_step_timing in s: 14.04 | consumed_samples: 28
ataset_32k/0 [default0]:Training epoch 0, iteration 7/999 | lr: 1.569e-05 | global_batch_size: 4 | global_step: 7 | reduced_train_loss: 0.5466 | train_step_timing in s: 13.87 | consumed_samples: 32
ataset_32k/0 [default0]:Training epoch 0, iteration 8/999 | lr: 1.765e-05 | global_batch_size: 4 | global_step: 8 | reduced_train_loss: 0.6338 | train_step_timing in s: 13.79 | consumed_samples: 36
ataset_32k/0 [default0]:Training epoch 0, iteration 9/999 | lr: 1.961e-05 | global_batch_size: 4 | global_step: 9 | reduced_train_loss: 0.5959 | train_step_timing in s: 13.51 | consumed_samples: 40
ataset_32k/0 [default0]:Training epoch 0, iteration 10/999 | lr: 2.157e-05 | global_batch_size: 4 | global_step: 10 | reduced_train_loss: 0.5383 | train_step_timing in s: 14.04 | consumed_samples: 44
ataset_32k/0 [default0]:Training epoch 0, iteration 11/999 | lr: 2.353e-05 | global_batch_size: 4 | global_step: 11 | reduced_train_loss: 0.5589 | train_step_timing in s: 13.57 | consumed_samples: 48
ataset_32k/0 [default0]:Training epoch 0, iteration 12/999 | lr: 2.549e-05 | global_batch_size: 4 | global_step: 12 | reduced_train_loss: 0.5441 | train_step_timing in s: 14.05 | consumed_samples: 52
ataset_32k/0 [default0]:Training epoch 0, iteration 13/999 | lr: 2.745e-05 | global_batch_size: 4 | global_step: 13 | reduced_train_loss: 0.5335 | train_step_timing in s: 13.54 | consumed_samples: 56
ataset_32k/0 [default0]:Training epoch 0, iteration 14/999 | lr: 2.941e-05 | global_batch_size: 4 | global_step: 14 | reduced_train_loss: 0.4445 | train_step_timing in s: 14.03 | consumed_samples: 60
ataset_32k/0 [default0]:Training epoch 0, iteration 15/999 | lr: 3.137e-05 | global_batch_size: 4 | global_step: 15 | reduced_train_loss: 0.6416 | train_step_timing in s: 13.59 | consumed_samples: 64
ataset_32k/0 [default0]:Training epoch 0, iteration 16/999 | lr: 3.333e-05 | global_batch_size: 4 | global_step: 16 | reduced_train_loss: 0.4959 | train_step_timing in s: 14.1 | consumed_samples: 68
ataset_32k/0 [default0]:Training epoch 0, iteration 17/999 | lr: 3.529e-05 | global_batch_size: 4 | global_step: 17 | reduced_train_loss: 0.5062 | train_step_timing in s: 13.67 | consumed_samples: 72
ataset_32k/0 [default0]:Training epoch 0, iteration 18/999 | lr: 3.725e-05 | global_batch_size: 4 | global_step: 18 | reduced_train_loss: 0.3002 | train_step_timing in s: 13.86 | consumed_samples: 76
ataset_32k/0 [default0]:Training epoch 0, iteration 19/999 | lr: 3.922e-05 | global_batch_size: 4 | global_step: 19 | reduced_train_loss: 0.5253 | train_step_timing in s: 13.5 | consumed_samples: 80
ataset_32k/0 [default0]:Training epoch 0, iteration 20/999 | lr: 4.118e-05 | global_batch_size: 4 | global_step: 20 | reduced_train_loss: 0.6735 | train_step_timing in s: 13.83 | consumed_samples: 84
ataset_32k/0 [default0]:Training epoch 0, iteration 21/999 | lr: 4.314e-05 | global_batch_size: 4 | global_step: 21 | reduced_train_loss: 0.3593 | train_step_timing in s: 13.85 | consumed_samples: 88
ataset_32k/0 [default0]:Training epoch 0, iteration 22/999 | lr: 4.51e-05 | global_batch_size: 4 | global_step: 22 | reduced_train_loss: 0.5715 | train_step_timing in s: 13.86 | consumed_samples: 92
ataset_32k/0 [default0]:Training epoch 0, iteration 23/999 | lr: 4.706e-05 | global_batch_size: 4 | global_step: 23 | reduced_train_loss: 0.4452 | train_step_timing in s: 13.89 | consumed_samples: 96
ataset_32k/0 [default0]:Training epoch 0, iteration 24/999 | lr: 4.902e-05 | global_batch_size: 4 | global_step: 24 | reduced_train_loss: 0.5219 | train_step_timing in s: 13.92 | consumed_samples: 100
ataset_32k/0 [default0]:Training epoch 0, iteration 25/999 | lr: 5.098e-05 | global_batch_size: 4 | global_step: 25 | reduced_train_loss: 0.516 | train_step_timing in s: 13.76 | consumed_samples: 104
ataset_32k/0 [default0]:Training epoch 0, iteration 26/999 | lr: 5.294e-05 | global_batch_size: 4 | global_step: 26 | reduced_train_loss: 0.4207 | train_step_timing in s: 13.94 | consumed_samples: 108
ataset_32k/0 [default0]:Training epoch 0, iteration 27/999 | lr: 5.49e-05 | global_batch_size: 4 | global_step: 27 | reduced_train_loss: 0.4436 | train_step_timing in s: 13.79 | consumed_samples: 112
ataset_32k/0 [default0]:Training epoch 0, iteration 28/999 | lr: 5.686e-05 | global_batch_size: 4 | global_step: 28 | reduced_train_loss: 0.5416 | train_step_timing in s: 13.88 | consumed_samples: 116
ataset_32k/0 [default0]:Training epoch 0, iteration 29/999 | lr: 5.882e-05 | global_batch_size: 4 | global_step: 29 | reduced_train_loss: 0.6188 | train_step_timing in s: 13.78 | consumed_samples: 120
ataset_32k/0 [default0]:Validation: iteration 1/0
ataset_32k/0 [default0]:Validation: iteration 2/0
ataset_32k/0 [default0]:Validation: iteration 3/0
ataset_32k/0 [default0]:Validation: iteration 4/0
ataset_32k/0 [default0]:Validation: iteration 5/0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:39:23 pl:353] Using FullyParallelSaveStrategyWrapper(torch_dist, 1) dist-ckpt save strategy.
ataset_32k/0 [default1]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0 [default1]:  import pynvml  # type: ignore[import]
ataset_32k/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0 [default0]:  import pynvml  # type: ignore[import]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:39:28 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 30 : Start time: 1764077963.243s : Save duration: 5.389s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:39:29 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=29-consumed_samples=120.0-last.ckpt
ataset_32k/0 [default0]:Training epoch 0, iteration 30/999 | lr: 6.078e-05 | global_batch_size: 4 | global_step: 30 | reduced_train_loss: 0.4084 | train_step_timing in s: 14.46 | consumed_samples: 124 | val_loss: 0.5661
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:39:43 pl:222] Successfully saved checkpoint from iteration      30 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=29-consumed_samples=120.0-last.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:39:43 model_checkpoint:657] Async checkpoint save for step 30 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=29-consumed_samples=120.0-last.ckpt) finalized successfully.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:39:43 dist_ckpt_io:160] Async finalization time took 0.064 s
ataset_32k/0 [default0]:Training epoch 0, iteration 31/999 | lr: 6.275e-05 | global_batch_size: 4 | global_step: 31 | reduced_train_loss: 0.5113 | train_step_timing in s: 13.96 | consumed_samples: 128 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 32/999 | lr: 6.471e-05 | global_batch_size: 4 | global_step: 32 | reduced_train_loss: 0.6055 | train_step_timing in s: 14.15 | consumed_samples: 132 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 33/999 | lr: 6.667e-05 | global_batch_size: 4 | global_step: 33 | reduced_train_loss: 0.5125 | train_step_timing in s: 13.94 | consumed_samples: 136 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 34/999 | lr: 6.863e-05 | global_batch_size: 4 | global_step: 34 | reduced_train_loss: 0.6113 | train_step_timing in s: 14.11 | consumed_samples: 140 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 35/999 | lr: 7.059e-05 | global_batch_size: 4 | global_step: 35 | reduced_train_loss: 0.6651 | train_step_timing in s: 13.7 | consumed_samples: 144 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 36/999 | lr: 7.255e-05 | global_batch_size: 4 | global_step: 36 | reduced_train_loss: 0.3852 | train_step_timing in s: 14.08 | consumed_samples: 148 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 37/999 | lr: 7.451e-05 | global_batch_size: 4 | global_step: 37 | reduced_train_loss: 0.464 | train_step_timing in s: 13.76 | consumed_samples: 152 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 38/999 | lr: 7.647e-05 | global_batch_size: 4 | global_step: 38 | reduced_train_loss: 0.5624 | train_step_timing in s: 14.13 | consumed_samples: 156 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 39/999 | lr: 7.843e-05 | global_batch_size: 4 | global_step: 39 | reduced_train_loss: 0.5942 | train_step_timing in s: 13.67 | consumed_samples: 160 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 40/999 | lr: 8.039e-05 | global_batch_size: 4 | global_step: 40 | reduced_train_loss: 0.5444 | train_step_timing in s: 13.89 | consumed_samples: 164 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 41/999 | lr: 8.235e-05 | global_batch_size: 4 | global_step: 41 | reduced_train_loss: 0.7865 | train_step_timing in s: 13.62 | consumed_samples: 168 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 42/999 | lr: 8.431e-05 | global_batch_size: 4 | global_step: 42 | reduced_train_loss: 0.4729 | train_step_timing in s: 14.02 | consumed_samples: 172 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 43/999 | lr: 8.627e-05 | global_batch_size: 4 | global_step: 43 | reduced_train_loss: 0.5792 | train_step_timing in s: 13.56 | consumed_samples: 176 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 44/999 | lr: 8.824e-05 | global_batch_size: 4 | global_step: 44 | reduced_train_loss: 0.5069 | train_step_timing in s: 13.98 | consumed_samples: 180 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 45/999 | lr: 9.02e-05 | global_batch_size: 4 | global_step: 45 | reduced_train_loss: 0.4615 | train_step_timing in s: 13.86 | consumed_samples: 184 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 46/999 | lr: 9.216e-05 | global_batch_size: 4 | global_step: 46 | reduced_train_loss: 0.5985 | train_step_timing in s: 13.8 | consumed_samples: 188 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 47/999 | lr: 9.412e-05 | global_batch_size: 4 | global_step: 47 | reduced_train_loss: 0.3774 | train_step_timing in s: 13.68 | consumed_samples: 192 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 48/999 | lr: 9.608e-05 | global_batch_size: 4 | global_step: 48 | reduced_train_loss: 0.4674 | train_step_timing in s: 14.08 | consumed_samples: 196 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 49/999 | lr: 9.804e-05 | global_batch_size: 4 | global_step: 49 | reduced_train_loss: 0.5681 | train_step_timing in s: 13.65 | consumed_samples: 200 | val_loss: 0.5661
ataset_32k/0 [default0]:Epoch 0, global step 49: 'val_loss' reached 0.56608 (best 0.56608), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=49-consumed_samples=200.0.ckpt' as top 2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:44:09 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 49 : Start time: 1764078247.600s : Save duration: 1.632s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:44:09 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=49-consumed_samples=200.0.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:44:09 dist_ckpt_io:160] Async finalization time took 0.002 s
ataset_32k/0 [default0]:Training epoch 0, iteration 50/999 | lr: 0.0001 | global_batch_size: 4 | global_step: 50 | reduced_train_loss: 0.2583 | train_step_timing in s: 13.99 | consumed_samples: 204 | val_loss: 0.5661
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:44:23 pl:222] Successfully saved checkpoint from iteration      49 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=49-consumed_samples=200.0.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:44:23 model_checkpoint:657] Async checkpoint save for step 50 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=49-consumed_samples=200.0.ckpt) finalized successfully.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:44:23 dist_ckpt_io:160] Async finalization time took 0.158 s
ataset_32k/0 [default0]:Training epoch 0, iteration 51/999 | lr: 0.0001 | global_batch_size: 4 | global_step: 51 | reduced_train_loss: 0.4398 | train_step_timing in s: 13.93 | consumed_samples: 208 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 52/999 | lr: 0.0001 | global_batch_size: 4 | global_step: 52 | reduced_train_loss: 0.4006 | train_step_timing in s: 14.03 | consumed_samples: 212 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 53/999 | lr: 0.0001 | global_batch_size: 4 | global_step: 53 | reduced_train_loss: 0.5179 | train_step_timing in s: 13.92 | consumed_samples: 216 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 54/999 | lr: 0.0001 | global_batch_size: 4 | global_step: 54 | reduced_train_loss: 0.5027 | train_step_timing in s: 14.14 | consumed_samples: 220 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 55/999 | lr: 9.999e-05 | global_batch_size: 4 | global_step: 55 | reduced_train_loss: 0.4734 | train_step_timing in s: 13.72 | consumed_samples: 224 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 56/999 | lr: 9.999e-05 | global_batch_size: 4 | global_step: 56 | reduced_train_loss: 0.4272 | train_step_timing in s: 13.97 | consumed_samples: 228 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 57/999 | lr: 9.999e-05 | global_batch_size: 4 | global_step: 57 | reduced_train_loss: 0.5373 | train_step_timing in s: 13.58 | consumed_samples: 232 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 58/999 | lr: 9.998e-05 | global_batch_size: 4 | global_step: 58 | reduced_train_loss: 0.5753 | train_step_timing in s: 13.87 | consumed_samples: 236 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 59/999 | lr: 9.998e-05 | global_batch_size: 4 | global_step: 59 | reduced_train_loss: 0.353 | train_step_timing in s: 13.94 | consumed_samples: 240 | val_loss: 0.5661
ataset_32k/0 [default0]:Validation: iteration 1/0
ataset_32k/0 [default0]:Validation: iteration 2/0
ataset_32k/0 [default0]:Validation: iteration 3/0
ataset_32k/0 [default0]:Validation: iteration 4/0
ataset_32k/0 [default0]:Validation: iteration 5/0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:50:57 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 60 : Start time: 1764078657.021s : Save duration: 0.313s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:50:57 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=59-consumed_samples=240.0-last.ckpt
ataset_32k/0 [default0]:Training epoch 0, iteration 60/999 | lr: 9.997e-05 | global_batch_size: 4 | global_step: 60 | reduced_train_loss: 0.4517 | train_step_timing in s: 14.21 | consumed_samples: 244 | val_loss: 0.5661
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:51:12 pl:222] Successfully saved checkpoint from iteration      60 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=59-consumed_samples=240.0-last.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:51:12 model_checkpoint:657] Async checkpoint save for step 60 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=59-consumed_samples=240.0-last.ckpt) finalized successfully.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:51:12 dist_ckpt_io:160] Async finalization time took 0.562 s
ataset_32k/0 [default0]:Training epoch 0, iteration 61/999 | lr: 9.997e-05 | global_batch_size: 4 | global_step: 61 | reduced_train_loss: 0.6674 | train_step_timing in s: 13.53 | consumed_samples: 248 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 62/999 | lr: 9.996e-05 | global_batch_size: 4 | global_step: 62 | reduced_train_loss: 0.5553 | train_step_timing in s: 14.04 | consumed_samples: 252 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 63/999 | lr: 9.995e-05 | global_batch_size: 4 | global_step: 63 | reduced_train_loss: 0.4701 | train_step_timing in s: 13.88 | consumed_samples: 256 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 64/999 | lr: 9.995e-05 | global_batch_size: 4 | global_step: 64 | reduced_train_loss: 0.4875 | train_step_timing in s: 14.03 | consumed_samples: 260 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 65/999 | lr: 9.994e-05 | global_batch_size: 4 | global_step: 65 | reduced_train_loss: 0.4673 | train_step_timing in s: 13.88 | consumed_samples: 264 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 66/999 | lr: 9.993e-05 | global_batch_size: 4 | global_step: 66 | reduced_train_loss: 0.4822 | train_step_timing in s: 14.07 | consumed_samples: 268 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 67/999 | lr: 9.992e-05 | global_batch_size: 4 | global_step: 67 | reduced_train_loss: 0.4877 | train_step_timing in s: 13.65 | consumed_samples: 272 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 68/999 | lr: 9.991e-05 | global_batch_size: 4 | global_step: 68 | reduced_train_loss: 0.523 | train_step_timing in s: 13.77 | consumed_samples: 276 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 69/999 | lr: 9.99e-05 | global_batch_size: 4 | global_step: 69 | reduced_train_loss: 0.3866 | train_step_timing in s: 13.7 | consumed_samples: 280 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 70/999 | lr: 9.989e-05 | global_batch_size: 4 | global_step: 70 | reduced_train_loss: 0.5591 | train_step_timing in s: 13.78 | consumed_samples: 284 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 71/999 | lr: 9.988e-05 | global_batch_size: 4 | global_step: 71 | reduced_train_loss: 0.5474 | train_step_timing in s: 13.75 | consumed_samples: 288 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 72/999 | lr: 9.987e-05 | global_batch_size: 4 | global_step: 72 | reduced_train_loss: 0.3855 | train_step_timing in s: 13.88 | consumed_samples: 292 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 73/999 | lr: 9.986e-05 | global_batch_size: 4 | global_step: 73 | reduced_train_loss: 0.4418 | train_step_timing in s: 13.89 | consumed_samples: 296 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 74/999 | lr: 9.984e-05 | global_batch_size: 4 | global_step: 74 | reduced_train_loss: 0.6318 | train_step_timing in s: 14.06 | consumed_samples: 300 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 75/999 | lr: 9.983e-05 | global_batch_size: 4 | global_step: 75 | reduced_train_loss: 0.5816 | train_step_timing in s: 13.86 | consumed_samples: 304 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 76/999 | lr: 9.982e-05 | global_batch_size: 4 | global_step: 76 | reduced_train_loss: 0.7629 | train_step_timing in s: 13.71 | consumed_samples: 308 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 77/999 | lr: 9.98e-05 | global_batch_size: 4 | global_step: 77 | reduced_train_loss: 0.5554 | train_step_timing in s: 13.81 | consumed_samples: 312 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 78/999 | lr: 9.979e-05 | global_batch_size: 4 | global_step: 78 | reduced_train_loss: 0.4572 | train_step_timing in s: 13.73 | consumed_samples: 316 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 79/999 | lr: 9.977e-05 | global_batch_size: 4 | global_step: 79 | reduced_train_loss: 0.6707 | train_step_timing in s: 13.55 | consumed_samples: 320 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 80/999 | lr: 9.975e-05 | global_batch_size: 4 | global_step: 80 | reduced_train_loss: 0.6123 | train_step_timing in s: 13.94 | consumed_samples: 324 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 81/999 | lr: 9.974e-05 | global_batch_size: 4 | global_step: 81 | reduced_train_loss: 0.382 | train_step_timing in s: 13.73 | consumed_samples: 328 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 82/999 | lr: 9.972e-05 | global_batch_size: 4 | global_step: 82 | reduced_train_loss: 0.4741 | train_step_timing in s: 13.89 | consumed_samples: 332 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 83/999 | lr: 9.97e-05 | global_batch_size: 4 | global_step: 83 | reduced_train_loss: 0.5912 | train_step_timing in s: 13.88 | consumed_samples: 336 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 84/999 | lr: 9.968e-05 | global_batch_size: 4 | global_step: 84 | reduced_train_loss: 0.6654 | train_step_timing in s: 13.87 | consumed_samples: 340 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 85/999 | lr: 9.967e-05 | global_batch_size: 4 | global_step: 85 | reduced_train_loss: 0.4538 | train_step_timing in s: 13.95 | consumed_samples: 344 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 86/999 | lr: 9.965e-05 | global_batch_size: 4 | global_step: 86 | reduced_train_loss: 0.4905 | train_step_timing in s: 13.87 | consumed_samples: 348 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 87/999 | lr: 9.963e-05 | global_batch_size: 4 | global_step: 87 | reduced_train_loss: 0.4523 | train_step_timing in s: 13.87 | consumed_samples: 352 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 88/999 | lr: 9.961e-05 | global_batch_size: 4 | global_step: 88 | reduced_train_loss: 0.5987 | train_step_timing in s: 13.93 | consumed_samples: 356 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 89/999 | lr: 9.958e-05 | global_batch_size: 4 | global_step: 89 | reduced_train_loss: 0.5684 | train_step_timing in s: 13.89 | consumed_samples: 360 | val_loss: 0.5661
ataset_32k/0 [default0]:Validation: iteration 1/0
ataset_32k/0 [default0]:Validation: iteration 2/0
ataset_32k/0 [default0]:Validation: iteration 3/0
ataset_32k/0 [default0]:Validation: iteration 4/0
ataset_32k/0 [default0]:Validation: iteration 5/0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:02:21 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 90 : Start time: 1764079341.293s : Save duration: 0.314s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:02:21 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=89-consumed_samples=360.0-last.ckpt
ataset_32k/0 [default0]:Training epoch 0, iteration 90/999 | lr: 9.956e-05 | global_batch_size: 4 | global_step: 90 | reduced_train_loss: 1.054 | train_step_timing in s: 13.5 | consumed_samples: 364 | val_loss: 0.5661
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:02:35 pl:222] Successfully saved checkpoint from iteration      90 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=89-consumed_samples=360.0-last.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:02:35 model_checkpoint:657] Async checkpoint save for step 90 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=89-consumed_samples=360.0-last.ckpt) finalized successfully.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:02:35 dist_ckpt_io:160] Async finalization time took 0.166 s
ataset_32k/0 [default0]:Training epoch 0, iteration 91/999 | lr: 9.954e-05 | global_batch_size: 4 | global_step: 91 | reduced_train_loss: 0.3681 | train_step_timing in s: 13.77 | consumed_samples: 368 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 92/999 | lr: 9.952e-05 | global_batch_size: 4 | global_step: 92 | reduced_train_loss: 0.7324 | train_step_timing in s: 13.93 | consumed_samples: 372 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 93/999 | lr: 9.95e-05 | global_batch_size: 4 | global_step: 93 | reduced_train_loss: 0.5377 | train_step_timing in s: 13.76 | consumed_samples: 376 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 94/999 | lr: 9.947e-05 | global_batch_size: 4 | global_step: 94 | reduced_train_loss: 0.6909 | train_step_timing in s: 13.88 | consumed_samples: 380 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 95/999 | lr: 9.945e-05 | global_batch_size: 4 | global_step: 95 | reduced_train_loss: 0.7578 | train_step_timing in s: 13.5 | consumed_samples: 384 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 96/999 | lr: 9.942e-05 | global_batch_size: 4 | global_step: 96 | reduced_train_loss: 0.5846 | train_step_timing in s: 13.94 | consumed_samples: 388 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 97/999 | lr: 9.94e-05 | global_batch_size: 4 | global_step: 97 | reduced_train_loss: 0.5372 | train_step_timing in s: 13.7 | consumed_samples: 392 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 98/999 | lr: 9.937e-05 | global_batch_size: 4 | global_step: 98 | reduced_train_loss: 0.5064 | train_step_timing in s: 13.94 | consumed_samples: 396 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 99/999 | lr: 9.935e-05 | global_batch_size: 4 | global_step: 99 | reduced_train_loss: 0.5786 | train_step_timing in s: 13.89 | consumed_samples: 400 | val_loss: 0.5661
ataset_32k/0 [default0]:Epoch 0, global step 99: 'val_loss' reached 0.56608 (best 0.56608), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=99-consumed_samples=400.0.ckpt' as top 2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:04:40 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 99 : Start time: 1764079480.163s : Save duration: 0.317s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:04:40 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=99-consumed_samples=400.0.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:04:40 dist_ckpt_io:160] Async finalization time took 0.002 s
ataset_32k/0 [default0]:Training epoch 0, iteration 100/999 | lr: 9.932e-05 | global_batch_size: 4 | global_step: 100 | reduced_train_loss: 0.335 | train_step_timing in s: 14.18 | consumed_samples: 404 | val_loss: 0.5661
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:04:55 pl:222] Successfully saved checkpoint from iteration      99 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=99-consumed_samples=400.0.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:04:55 model_checkpoint:657] Async checkpoint save for step 100 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=99-consumed_samples=400.0.ckpt) finalized successfully.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:04:55 dist_ckpt_io:160] Async finalization time took 0.630 s
ataset_32k/0 [default0]:Training epoch 0, iteration 101/999 | lr: 9.929e-05 | global_batch_size: 4 | global_step: 101 | reduced_train_loss: 0.3978 | train_step_timing in s: 13.84 | consumed_samples: 408 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 102/999 | lr: 9.926e-05 | global_batch_size: 4 | global_step: 102 | reduced_train_loss: 1.035 | train_step_timing in s: 13.87 | consumed_samples: 412 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 103/999 | lr: 9.923e-05 | global_batch_size: 4 | global_step: 103 | reduced_train_loss: 0.5643 | train_step_timing in s: 13.66 | consumed_samples: 416 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 104/999 | lr: 9.92e-05 | global_batch_size: 4 | global_step: 104 | reduced_train_loss: 0.3365 | train_step_timing in s: 14.12 | consumed_samples: 420 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 105/999 | lr: 9.918e-05 | global_batch_size: 4 | global_step: 105 | reduced_train_loss: 0.4684 | train_step_timing in s: 13.65 | consumed_samples: 424 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 106/999 | lr: 9.915e-05 | global_batch_size: 4 | global_step: 106 | reduced_train_loss: 0.4146 | train_step_timing in s: 14.03 | consumed_samples: 428 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 107/999 | lr: 9.911e-05 | global_batch_size: 4 | global_step: 107 | reduced_train_loss: 0.7036 | train_step_timing in s: 13.54 | consumed_samples: 432 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 108/999 | lr: 9.908e-05 | global_batch_size: 4 | global_step: 108 | reduced_train_loss: 0.4706 | train_step_timing in s: 14.11 | consumed_samples: 436 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 109/999 | lr: 9.905e-05 | global_batch_size: 4 | global_step: 109 | reduced_train_loss: 0.669 | train_step_timing in s: 13.76 | consumed_samples: 440 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 110/999 | lr: 9.902e-05 | global_batch_size: 4 | global_step: 110 | reduced_train_loss: 0.4526 | train_step_timing in s: 14.15 | consumed_samples: 444 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 111/999 | lr: 9.899e-05 | global_batch_size: 4 | global_step: 111 | reduced_train_loss: 0.4312 | train_step_timing in s: 13.54 | consumed_samples: 448 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 112/999 | lr: 9.895e-05 | global_batch_size: 4 | global_step: 112 | reduced_train_loss: 0.3911 | train_step_timing in s: 14.09 | consumed_samples: 452 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 113/999 | lr: 9.892e-05 | global_batch_size: 4 | global_step: 113 | reduced_train_loss: 0.4604 | train_step_timing in s: 14.03 | consumed_samples: 456 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 114/999 | lr: 9.888e-05 | global_batch_size: 4 | global_step: 114 | reduced_train_loss: 0.455 | train_step_timing in s: 13.86 | consumed_samples: 460 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 115/999 | lr: 9.885e-05 | global_batch_size: 4 | global_step: 115 | reduced_train_loss: 0.428 | train_step_timing in s: 13.41 | consumed_samples: 464 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 116/999 | lr: 9.881e-05 | global_batch_size: 4 | global_step: 116 | reduced_train_loss: 0.6698 | train_step_timing in s: 13.74 | consumed_samples: 468 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 117/999 | lr: 9.878e-05 | global_batch_size: 4 | global_step: 117 | reduced_train_loss: 0.6391 | train_step_timing in s: 13.6 | consumed_samples: 472 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 118/999 | lr: 9.874e-05 | global_batch_size: 4 | global_step: 118 | reduced_train_loss: 0.4502 | train_step_timing in s: 14.02 | consumed_samples: 476 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 119/999 | lr: 9.87e-05 | global_batch_size: 4 | global_step: 119 | reduced_train_loss: 0.5284 | train_step_timing in s: 13.8 | consumed_samples: 480 | val_loss: 0.5661
ataset_32k/0 [default0]:Validation: iteration 1/0
ataset_32k/0 [default0]:Validation: iteration 2/0
ataset_32k/0 [default0]:Validation: iteration 3/0
ataset_32k/0 [default0]:Validation: iteration 4/0
ataset_32k/0 [default0]:Validation: iteration 5/0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:13:48 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 120 : Start time: 1764080027.399s : Save duration: 0.708s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:13:48 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=119-consumed_samples=480.0-last.ckpt
ataset_32k/0 [default0]:Training epoch 0, iteration 120/999 | lr: 9.867e-05 | global_batch_size: 4 | global_step: 120 | reduced_train_loss: 0.9307 | train_step_timing in s: 13.95 | consumed_samples: 484 | val_loss: 0.5661
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:14:02 pl:222] Successfully saved checkpoint from iteration     120 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=119-consumed_samples=480.0-last.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:14:02 model_checkpoint:657] Async checkpoint save for step 120 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=119-consumed_samples=480.0-last.ckpt) finalized successfully.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:14:02 dist_ckpt_io:160] Async finalization time took 0.071 s
ataset_32k/0 [default0]:Training epoch 0, iteration 121/999 | lr: 9.863e-05 | global_batch_size: 4 | global_step: 121 | reduced_train_loss: 0.5273 | train_step_timing in s: 13.55 | consumed_samples: 488 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 122/999 | lr: 9.859e-05 | global_batch_size: 4 | global_step: 122 | reduced_train_loss: 0.6153 | train_step_timing in s: 13.85 | consumed_samples: 492 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 123/999 | lr: 9.855e-05 | global_batch_size: 4 | global_step: 123 | reduced_train_loss: 0.474 | train_step_timing in s: 13.94 | consumed_samples: 496 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 124/999 | lr: 9.851e-05 | global_batch_size: 4 | global_step: 124 | reduced_train_loss: 0.6079 | train_step_timing in s: 13.94 | consumed_samples: 500 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 125/999 | lr: 9.847e-05 | global_batch_size: 4 | global_step: 125 | reduced_train_loss: 0.55 | train_step_timing in s: 13.86 | consumed_samples: 504 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 126/999 | lr: 9.843e-05 | global_batch_size: 4 | global_step: 126 | reduced_train_loss: 0.4594 | train_step_timing in s: 14.0 | consumed_samples: 508 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 127/999 | lr: 9.839e-05 | global_batch_size: 4 | global_step: 127 | reduced_train_loss: 0.3833 | train_step_timing in s: 13.92 | consumed_samples: 512 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 128/999 | lr: 9.835e-05 | global_batch_size: 4 | global_step: 128 | reduced_train_loss: 0.5396 | train_step_timing in s: 14.11 | consumed_samples: 516 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 129/999 | lr: 9.83e-05 | global_batch_size: 4 | global_step: 129 | reduced_train_loss: 0.5086 | train_step_timing in s: 13.93 | consumed_samples: 520 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 130/999 | lr: 9.826e-05 | global_batch_size: 4 | global_step: 130 | reduced_train_loss: 0.4608 | train_step_timing in s: 14.12 | consumed_samples: 524 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 131/999 | lr: 9.822e-05 | global_batch_size: 4 | global_step: 131 | reduced_train_loss: 0.5796 | train_step_timing in s: 13.5 | consumed_samples: 528 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 132/999 | lr: 9.817e-05 | global_batch_size: 4 | global_step: 132 | reduced_train_loss: 0.6185 | train_step_timing in s: 14.11 | consumed_samples: 532 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 133/999 | lr: 9.813e-05 | global_batch_size: 4 | global_step: 133 | reduced_train_loss: 0.5197 | train_step_timing in s: 13.63 | consumed_samples: 536 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 134/999 | lr: 9.808e-05 | global_batch_size: 4 | global_step: 134 | reduced_train_loss: 0.5373 | train_step_timing in s: 14.02 | consumed_samples: 540 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 135/999 | lr: 9.804e-05 | global_batch_size: 4 | global_step: 135 | reduced_train_loss: 0.608 | train_step_timing in s: 13.88 | consumed_samples: 544 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 136/999 | lr: 9.799e-05 | global_batch_size: 4 | global_step: 136 | reduced_train_loss: 0.5515 | train_step_timing in s: 13.98 | consumed_samples: 548 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 137/999 | lr: 9.794e-05 | global_batch_size: 4 | global_step: 137 | reduced_train_loss: 0.6002 | train_step_timing in s: 13.9 | consumed_samples: 552 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 138/999 | lr: 9.79e-05 | global_batch_size: 4 | global_step: 138 | reduced_train_loss: 0.3651 | train_step_timing in s: 13.88 | consumed_samples: 556 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 139/999 | lr: 9.785e-05 | global_batch_size: 4 | global_step: 139 | reduced_train_loss: 0.7818 | train_step_timing in s: 13.52 | consumed_samples: 560 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 140/999 | lr: 9.78e-05 | global_batch_size: 4 | global_step: 140 | reduced_train_loss: 0.4719 | train_step_timing in s: 14.01 | consumed_samples: 564 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 141/999 | lr: 9.775e-05 | global_batch_size: 4 | global_step: 141 | reduced_train_loss: 0.4543 | train_step_timing in s: 13.9 | consumed_samples: 568 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 142/999 | lr: 9.77e-05 | global_batch_size: 4 | global_step: 142 | reduced_train_loss: 0.55 | train_step_timing in s: 14.11 | consumed_samples: 572 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 143/999 | lr: 9.765e-05 | global_batch_size: 4 | global_step: 143 | reduced_train_loss: 0.4996 | train_step_timing in s: 13.89 | consumed_samples: 576 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 144/999 | lr: 9.76e-05 | global_batch_size: 4 | global_step: 144 | reduced_train_loss: 0.4402 | train_step_timing in s: 14.06 | consumed_samples: 580 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 145/999 | lr: 9.755e-05 | global_batch_size: 4 | global_step: 145 | reduced_train_loss: 0.4298 | train_step_timing in s: 13.85 | consumed_samples: 584 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 146/999 | lr: 9.75e-05 | global_batch_size: 4 | global_step: 146 | reduced_train_loss: 0.3953 | train_step_timing in s: 14.09 | consumed_samples: 588 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 147/999 | lr: 9.745e-05 | global_batch_size: 4 | global_step: 147 | reduced_train_loss: 0.5681 | train_step_timing in s: 13.81 | consumed_samples: 592 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 148/999 | lr: 9.74e-05 | global_batch_size: 4 | global_step: 148 | reduced_train_loss: 0.4743 | train_step_timing in s: 13.82 | consumed_samples: 596 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 149/999 | lr: 9.734e-05 | global_batch_size: 4 | global_step: 149 | reduced_train_loss: 0.6228 | train_step_timing in s: 13.91 | consumed_samples: 600 | val_loss: 0.5661
ataset_32k/0 [default0]:Epoch 0, global step 149: 'val_loss' was not in top 2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:20:46 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 149 : Start time: 1764080446.155s : Save duration: 0.313s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:20:46 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=149-consumed_samples=600.0-last.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:20:46 dist_ckpt_io:160] Async finalization time took 0.001 s
ataset_32k/0 [default0]:Validation: iteration 1/0
ataset_32k/0 [default0]:Validation: iteration 2/0
ataset_32k/0 [default0]:Validation: iteration 3/0
ataset_32k/0 [default0]:Validation: iteration 4/0
ataset_32k/0 [default0]:Validation: iteration 5/0
ataset_32k/0 [default0]:Training epoch 0, iteration 150/999 | lr: 9.729e-05 | global_batch_size: 4 | global_step: 150 | reduced_train_loss: 0.6112 | train_step_timing in s: 13.81 | consumed_samples: 604 | val_loss: 0.5661
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:25:27 pl:222] Successfully saved checkpoint from iteration     149 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=149-consumed_samples=600.0-last.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:25:27 model_checkpoint:657] Async checkpoint save for step 150 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=149-consumed_samples=600.0-last.ckpt) finalized successfully.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:25:28 dist_ckpt_io:160] Async finalization time took 0.155 s
ataset_32k/0 [default0]:Training epoch 0, iteration 151/999 | lr: 9.724e-05 | global_batch_size: 4 | global_step: 151 | reduced_train_loss: 0.4968 | train_step_timing in s: 13.8 | consumed_samples: 608 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 152/999 | lr: 9.718e-05 | global_batch_size: 4 | global_step: 152 | reduced_train_loss: 0.6069 | train_step_timing in s: 13.8 | consumed_samples: 612 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 153/999 | lr: 9.713e-05 | global_batch_size: 4 | global_step: 153 | reduced_train_loss: 0.6592 | train_step_timing in s: 13.88 | consumed_samples: 616 | val_loss: 0.5661[default0]:
ataset_32k/0 [default0]:Training epoch 0, iteration 154/999 | lr: 9.707e-05 | global_batch_size: 4 | global_step: 154 | reduced_train_loss: 0.8393 | train_step_timing in s: 13.88 | consumed_samples: 620 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 155/999 | lr: 9.702e-05 | global_batch_size: 4 | global_step: 155 | reduced_train_loss: 0.439 | train_step_timing in s: 13.36 | consumed_samples: 624 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 156/999 | lr: 9.696e-05 | global_batch_size: 4 | global_step: 156 | reduced_train_loss: 0.5408 | train_step_timing in s: 13.85 | consumed_samples: 628 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 157/999 | lr: 9.69e-05 | global_batch_size: 4 | global_step: 157 | reduced_train_loss: 0.5504 | train_step_timing in s: 13.62 | consumed_samples: 632 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 158/999 | lr: 9.684e-05 | global_batch_size: 4 | global_step: 158 | reduced_train_loss: 0.685 | train_step_timing in s: 14.06 | consumed_samples: 636 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 159/999 | lr: 9.679e-05 | global_batch_size: 4 | global_step: 159 | reduced_train_loss: 0.5157 | train_step_timing in s: 13.53 | consumed_samples: 640 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 160/999 | lr: 9.673e-05 | global_batch_size: 4 | global_step: 160 | reduced_train_loss: 0.5745 | train_step_timing in s: 13.9 | consumed_samples: 644 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 161/999 | lr: 9.667e-05 | global_batch_size: 4 | global_step: 161 | reduced_train_loss: 0.4182 | train_step_timing in s: 13.57 | consumed_samples: 648 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 162/999 | lr: 9.661e-05 | global_batch_size: 4 | global_step: 162 | reduced_train_loss: 0.4656 | train_step_timing in s: 14.05 | consumed_samples: 652 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 163/999 | lr: 9.655e-05 | global_batch_size: 4 | global_step: 163 | reduced_train_loss: 0.5697 | train_step_timing in s: 13.64 | consumed_samples: 656 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 164/999 | lr: 9.649e-05 | global_batch_size: 4 | global_step: 164 | reduced_train_loss: 1.011 | train_step_timing in s: 13.88 | consumed_samples: 660 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 165/999 | lr: 9.643e-05 | global_batch_size: 4 | global_step: 165 | reduced_train_loss: 0.8011 | train_step_timing in s: 10.53 | consumed_samples: 664 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 166/999 | lr: 9.637e-05 | global_batch_size: 4 | global_step: 166 | reduced_train_loss: 0.4401 | train_step_timing in s: 14.09 | consumed_samples: 668 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 167/999 | lr: 9.63e-05 | global_batch_size: 4 | global_step: 167 | reduced_train_loss: 0.5356 | train_step_timing in s: 13.94 | consumed_samples: 672 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 168/999 | lr: 9.624e-05 | global_batch_size: 4 | global_step: 168 | reduced_train_loss: 0.5515 | train_step_timing in s: 13.72 | consumed_samples: 676 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 169/999 | lr: 9.618e-05 | global_batch_size: 4 | global_step: 169 | reduced_train_loss: 0.4183 | train_step_timing in s: 13.88 | consumed_samples: 680 | val_loss: 0.5661[default0]:
ataset_32k/0 [default0]:Training epoch 0, iteration 170/999 | lr: 9.611e-05 | global_batch_size: 4 | global_step: 170 | reduced_train_loss: 0.5925 | train_step_timing in s: 13.95 | consumed_samples: 684 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 171/999 | lr: 9.605e-05 | global_batch_size: 4 | global_step: 171 | reduced_train_loss: 0.4759 | train_step_timing in s: 13.71 | consumed_samples: 688 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 172/999 | lr: 9.599e-05 | global_batch_size: 4 | global_step: 172 | reduced_train_loss: 0.6435 | train_step_timing in s: 13.92 | consumed_samples: 692 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 173/999 | lr: 9.592e-05 | global_batch_size: 4 | global_step: 173 | reduced_train_loss: 0.5362 | train_step_timing in s: 13.9 | consumed_samples: 696 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 174/999 | lr: 9.585e-05 | global_batch_size: 4 | global_step: 174 | reduced_train_loss: 0.5285 | train_step_timing in s: 13.88 | consumed_samples: 700 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 175/999 | lr: 9.579e-05 | global_batch_size: 4 | global_step: 175 | reduced_train_loss: 0.558 | train_step_timing in s: 13.89 | consumed_samples: 704 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 176/999 | lr: 9.572e-05 | global_batch_size: 4 | global_step: 176 | reduced_train_loss: 0.6162 | train_step_timing in s: 13.97 | consumed_samples: 708 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 177/999 | lr: 9.565e-05 | global_batch_size: 4 | global_step: 177 | reduced_train_loss: 0.6793 | train_step_timing in s: 13.56 | consumed_samples: 712 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 178/999 | lr: 9.559e-05 | global_batch_size: 4 | global_step: 178 | reduced_train_loss: 0.6427 | train_step_timing in s: 13.94 | consumed_samples: 716 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 179/999 | lr: 9.552e-05 | global_batch_size: 4 | global_step: 179 | reduced_train_loss: 0.5415 | train_step_timing in s: 13.75 | consumed_samples: 720 | val_loss: 0.5661
ataset_32k/0 [default0]:Validation: iteration 1/0
ataset_32k/0 [default0]:Validation: iteration 2/0
ataset_32k/0 [default0]:Validation: iteration 3/0
ataset_32k/0 [default0]:Validation: iteration 4/0
ataset_32k/0 [default0]:Validation: iteration 5/0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:36:34 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 180 : Start time: 1764081393.403s : Save duration: 0.775s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:36:34 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=179-consumed_samples=720.0-last.ckpt
ataset_32k/0 [default0]:Training epoch 0, iteration 180/999 | lr: 9.545e-05 | global_batch_size: 4 | global_step: 180 | reduced_train_loss: 0.3527 | train_step_timing in s: 14.19 | consumed_samples: 724 | val_loss: 0.5661[default0]:
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:36:48 pl:222] Successfully saved checkpoint from iteration     180 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=179-consumed_samples=720.0-last.ckpt
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:36:48 model_checkpoint:657] Async checkpoint save for step 180 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-27-21/checkpoints/model_name=0--val_loss=0.57-step=179-consumed_samples=720.0-last.ckpt) finalized successfully.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 15:36:48 dist_ckpt_io:160] Async finalization time took 0.166 s
ataset_32k/0 [default0]:Training epoch 0, iteration 181/999 | lr: 9.538e-05 | global_batch_size: 4 | global_step: 181 | reduced_train_loss: 0.5634 | train_step_timing in s: 13.94 | consumed_samples: 728 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 182/999 | lr: 9.531e-05 | global_batch_size: 4 | global_step: 182 | reduced_train_loss: 0.5859 | train_step_timing in s: 14.0 | consumed_samples: 732 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 183/999 | lr: 9.524e-05 | global_batch_size: 4 | global_step: 183 | reduced_train_loss: 0.3846 | train_step_timing in s: 13.56 | consumed_samples: 736 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 184/999 | lr: 9.517e-05 | global_batch_size: 4 | global_step: 184 | reduced_train_loss: 0.7083 | train_step_timing in s: 13.89 | consumed_samples: 740 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 185/999 | lr: 9.51e-05 | global_batch_size: 4 | global_step: 185 | reduced_train_loss: 0.4405 | train_step_timing in s: 13.57 | consumed_samples: 744 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 186/999 | lr: 9.503e-05 | global_batch_size: 4 | global_step: 186 | reduced_train_loss: 0.4448 | train_step_timing in s: 13.98 | consumed_samples: 748 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 187/999 | lr: 9.496e-05 | global_batch_size: 4 | global_step: 187 | reduced_train_loss: 0.5512 | train_step_timing in s: 13.61 | consumed_samples: 752 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 188/999 | lr: 9.488e-05 | global_batch_size: 4 | global_step: 188 | reduced_train_loss: 0.8186 | train_step_timing in s: 13.71 | consumed_samples: 756 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 189/999 | lr: 9.481e-05 | global_batch_size: 4 | global_step: 189 | reduced_train_loss: 0.4769 | train_step_timing in s: 13.97 | consumed_samples: 760 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 190/999 | lr: 9.474e-05 | global_batch_size: 4 | global_step: 190 | reduced_train_loss: 0.3977 | train_step_timing in s: 13.95 | consumed_samples: 764 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 191/999 | lr: 9.466e-05 | global_batch_size: 4 | global_step: 191 | reduced_train_loss: 0.5989 | train_step_timing in s: 13.8 | consumed_samples: 768 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 192/999 | lr: 9.459e-05 | global_batch_size: 4 | global_step: 192 | reduced_train_loss: 0.4769 | train_step_timing in s: 13.88 | consumed_samples: 772 | val_loss: 0.5661
ataset_32k/0 [default0]:Training epoch 0, iteration 193/999 | lr: 9.451e-05 | global_batch_size: 4 | global_step: 193 | reduced_train_loss: 0.4735 | train_step_timing in s: 13.89 | consumed_samples: 776 | val_loss: 0.5661