[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/nemo/qwen3_600m_finetune_34691.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

python "$1"/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
─ Entering Experiment qwen3_600m_lora_finetune_crowd_code_dataset_32k with id… ─
[14:08:28] INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_lora_finetune_crowd_c                       
                    ode_dataset_32k/qwen3_600m_lora_finet                       
                    une_crowd_code_dataset_32k_1764076107                       
                    /qwen3_600m_lora_finetune_crowd_code_                       
                    dataset_32k                                                 
[14:08:28] Launching job                                       experiment.py:744
           qwen3_600m_lora_finetune_crowd_code_dataset_32k for                  
           experiment                                                           
           qwen3_600m_lora_finetune_crowd_code_dataset_32k                      
           INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_lora_finetune_crowd_c                       
                    ode_dataset_32k/qwen3_600m_lora_finet                       
                    une_crowd_code_dataset_32k_1764076107                       
                    /qwen3_600m_lora_finetune_crowd_code_                       
                    dataset_32k                                                 
           INFO     Launched app:                                launcher.py:111
                    local_persistent://nemo_run/qwen3_600m_lora_                
                    finetune_crowd_code_dataset_32k-bmpq0ggjszrj                
                    fd                                                          
─ Waiting for Experiment qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764… ─

Experiment Status for qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764076107

Task 0: qwen3_600m_lora_finetune_crowd_code_dataset_32k
- Status: RUNNING
- Executor: LocalExecutor
- Job id: qwen3_600m_lora_finetune_crowd_code_dataset_32k-bmpq0ggjszrjfd
- Local Directory: /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_lora_finetune_crowd_code_dataset_32k/qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764076107/qwen3_600m_lora_finetune_crowd_code_dataset_32k

           INFO     Waiting for job                              launcher.py:130
                    qwen3_600m_lora_finetune_crowd_code_dataset_                
                    32k-bmpq0ggjszrjfd to finish [log=True]...                  
ataset_32k/0 /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0   import pynvml  # type: ignore[import]
ataset_32k/0 I1125 14:08:30.616000 744382 torch/distributed/run.py:657] Using nproc_per_node=2.
ataset_32k/0 W1125 14:08:30.617000 744382 torch/distributed/run.py:774] 
ataset_32k/0 W1125 14:08:30.617000 744382 torch/distributed/run.py:774] *****************************************
ataset_32k/0 W1125 14:08:30.617000 744382 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
ataset_32k/0 W1125 14:08:30.617000 744382 torch/distributed/run.py:774] *****************************************
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199] Starting elastic_operator with launch configs:
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   entrypoint         : nemo_run.core.runners.fdl_runner
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   min_nodes          : 1
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   max_nodes          : 1
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   nproc_per_node     : 2
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   run_id             : 5289
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   rdzv_backend       : c10d
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   rdzv_endpoint      : localhost:0
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   rdzv_configs       : {'timeout': 900}
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   max_restarts       : 0
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   monitor_interval   : 0.1
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   log_dir            : /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_lora_finetune_crowd_code_dataset_32k/qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764076107/qwen3_600m_lora_finetune_crowd_code_dataset_32k/nemo_run/qwen3_600m_lora_finetune_crowd_code_dataset_32k-bmpq0ggjszrjfd/torchelastic/qwen3_600m_lora_finetune_crowd_code_dataset_32k
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   metrics_cfg        : {}
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199]   event_log_handler  : null
ataset_32k/0 I1125 14:08:30.621000 744382 torch/distributed/launcher/api.py:199] 
ataset_32k/0 I1125 14:08:30.627000 744382 torch/distributed/elastic/agent/server/api.py:869] [default] starting workers for entrypoint: python3
ataset_32k/0 I1125 14:08:30.628000 744382 torch/distributed/elastic/agent/server/api.py:677] [default] Rendezvous'ing worker group
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523] [default] Rendezvous complete for workers. Result:
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   restart_count=0
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   master_addr=hai004.haicore.berlin
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   master_port=39485
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   group_rank=0
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   group_world_size=1
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   local_ranks=[0, 1]
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   role_ranks=[0, 1]
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   global_ranks=[0, 1]
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   role_world_sizes=[2, 2]
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   global_world_sizes=[2, 2]
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523]   event_log_handler=null
ataset_32k/0 I1125 14:08:30.746000 744382 torch/distributed/elastic/agent/server/api.py:523] 
ataset_32k/0 I1125 14:08:30.747000 744382 torch/distributed/elastic/agent/server/api.py:685] [default] Starting worker group
ataset_32k/0 I1125 14:08:30.748000 744382 torch/distributed/elastic/agent/server/local_elastic_agent.py:296] use_agent_store: True
ataset_32k/0 I1125 14:08:30.749000 744382 torch/distributed/elastic/agent/server/local_elastic_agent.py:192] Environment variable 'TORCHELASTIC_ENABLE_FILE_TIMER' not found. Do not start FileTimerServer.
ataset_32k/0 I1125 14:08:30.749000 744382 torch/distributed/elastic/agent/server/local_elastic_agent.py:236] Environment variable 'TORCHELASTIC_HEALTH_CHECK_PORT' not found. Do not start health check.
ataset_32k/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0 [default0]:  import pynvml  # type: ignore[import]
ataset_32k/0 [default1]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0 [default1]:  import pynvml  # type: ignore[import]
ataset_32k/0 [default0]:trainer.num_devices: 2
ataset_32k/0 [default0]:trainer.num_nodes: 1
ataset_32k/0 [default0]:trainer.strategy.tensor_model_parallel_size: 2
ataset_32k/0 [default0]:GPU available: True (cuda), used: True
ataset_32k/0 [default0]:TPU available: False, using: 0 TPU cores
ataset_32k/0 [default0]:HPU available: False, using: 0 HPUs
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:08:53 nemo_logger:190] "update_logger_directory" is True. Overwriting tensorboard logger "save_dir" to /fast/project/HFMI_SynergyUnit/tab_model/data/outputs/tb_logs
ataset_32k/0 [default0]:trainer.strategy.pipeline_model_parallel_size: 1
ataset_32k/0 [default0]:trainer.strategy.context_parallel_size: 1
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:08:53 nemo_logger:209] The Trainer already contains a ModelCheckpoint callback. This will be overwritten.
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:08:53 nemo_logger:239] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
ataset_32k/0 [default0]:data.global_batch_size: 4
ataset_32k/0 [default0]:data.micro_batch_size: 1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:53 api:1251] Disabling try_restore_best_ckpt restoration for adapters
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:53 nemo_logger:162] Experiments will be logged at /fast/project/HFMI_SynergyUnit/tab_model/data/outputs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_14-08-53
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:53 megatron_strategy:460] Fixing mis-match between ddp-config & mcore-optimizer config
ataset_32k/0 [default0]:You are using a CUDA device ('NVIDIA H100 80GB HBM3') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:449] Rank 0 has data parallel group : [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:455] Rank 0 has combined group of data parallel and context parallel : [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:460] All data parallel group ranks with context parallel combined: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:463] Ranks 0 has data parallel rank: 0
ataset_32k/0 [default0]:[W1125 14:08:54.665972784 ProcessGroupNCCL.cpp:981] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
ataset_32k/0 [default0]:----------------------------------------------------------------------------------------------------
ataset_32k/0 [default0]:distributed_backend=nccl
ataset_32k/0 [default0]:All distributed processes registered. Starting with 2 processes
ataset_32k/0 [default0]:----------------------------------------------------------------------------------------------------
ataset_32k/0 [default0]:
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:471] Rank 0 has context parallel group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:474] All context parallel group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:475] Ranks 0 has context parallel rank: 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:482] Rank 0 has model parallel group: [0, 1]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:483] All model parallel group ranks: [[0, 1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:492] Rank 0 has tensor model parallel group: [0, 1]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:496] All tensor model parallel group ranks: [[0, 1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:497] Rank 0 has tensor model parallel rank: 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:518] All expert tensor parallel group ranks: [[0, 1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:519] Rank 0 has expert tensor parallel rank: 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:532] Rank 0 has pipeline model parallel group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:544] Rank 0 has embedding group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:550] All pipeline model parallel group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:551] Rank 0 has pipeline model parallel rank 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:552] All embedding group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:08:54 megatron_init:553] Rank 0 has embedding rank: 0
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default1]:trainer.num_devices: 2
ataset_32k/0 [default1]:trainer.num_nodes: 1
ataset_32k/0 [default1]:trainer.strategy.tensor_model_parallel_size: 2
ataset_32k/0 [default1]:trainer.strategy.pipeline_model_parallel_size: 1
ataset_32k/0 [default1]:trainer.strategy.context_parallel_size: 1
ataset_32k/0 [default1]:data.global_batch_size: 4
ataset_32k/0 [default1]:data.micro_batch_size: 1
ataset_32k/0 [default1]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default1]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default1]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
ataset_32k/0 [default1]:[W1125 14:08:59.348431161 ProcessGroupNCCL.cpp:981] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
ataset_32k/0 [default1]:[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:00 model_transform:67] Setting up ModelTransform for stage: fit
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:00 model_transform:70] Found model_transform attribute on pl_module
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:00 model_transform:73] Set model_transform to: <function _call_counter.<locals>.wrapper at 0x7f61603df760>
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:00 base:373] Use preset vocab_size: 151936, original vocab_size: 151669, dummy tokens: 267.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:00 megatron_strategy:472] Copying Trainer's 'max_steps' (1000) to LR scheduler's 'max_steps'.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:00 num_microbatches_calculator:228] setting number of microbatches to constant 4
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:00 megatron_strategy:1045] Doing selective restore from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:00 pl:411] Loaded sharded_state_dict_metadata from checkpoint: {'distrib_optim_sharding_type': 'fully_sharded_model_space'}
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:00 pl:275] Using <megatron.core.dist_checkpointing.strategies.fully_parallel.FullyParallelLoadStrategyWrapper object at 0x7f6183206440> dist-ckpt load strategy.
ataset_32k/0 [default1]:LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1]
ataset_32k/0 [default0]:LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:09:00 serialization:184] DEPRECATED: Passing 'checkpoint_dir' as a Path object in load_common_state_dict will no longer be supported in a future release. Please pass it as a string instead.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:01 pl:297] Global Checkpoint Load : Rank : 0 : Start time : 1764076140.806s : Time spent in load_checkpoint: 0.978s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:01 megatron_strategy:1050] Restoring model weights from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:01 megatron_strategy:1058] Finished restoring from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True), cleaning up.
ataset_32k/0 [default0]:
ataset_32k/0 [default0]:  | Name   | Type     | Params | Mode 
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:0 | module | GPTModel | 298 M  | train
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:298 M     Trainable params
ataset_32k/0 [default0]:0         Non-trainable params
ataset_32k/0 [default0]:298 M     Total params
ataset_32k/0 [default0]:1,192.231 Total estimated model params size (MB)
ataset_32k/0 [default0]:569       Modules in train mode
ataset_32k/0 [default0]:0         Modules in eval mode
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.0.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.0.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.0.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.0.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.1.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.1.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.1.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.1.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.2.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.2.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.2.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.2.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.3.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.3.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.3.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.3.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.4.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.4.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.4.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.4.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.5.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.5.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.5.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.5.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.6.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.6.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.6.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.6.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.7.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.7.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.7.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.7.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.8.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.8.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.8.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.8.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.9.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.9.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.9.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.9.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.10.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.10.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.10.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.10.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.11.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.11.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.11.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.11.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.12.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.12.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.12.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.12.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.13.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.13.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.13.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.13.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.14.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.14.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.14.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.14.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.15.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.15.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.15.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.15.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.16.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.16.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.16.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.16.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.17.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.17.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.17.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.17.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.18.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.18.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.18.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.18.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.19.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.19.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.19.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.19.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.20.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.20.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.20.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.20.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.21.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.21.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.21.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.21.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.22.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.22.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.22.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.22.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.23.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.23.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.23.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.23.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.24.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.24.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.24.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.24.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.25.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.25.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.25.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.25.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.26.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.26.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.26.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.26.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.27.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.27.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.27.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 lora:468] Adding lora to: module.decoder.layers.27.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 model_transform:97] After applying model_transform:
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:  | Name   | Type     | Params | Mode 
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:0 | module | GPTModel | 306 M  | train
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:8.7 M     Trainable params
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:298 M     Non-trainable params
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:306 M     Total params
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:1,227.096 Total estimated model params size (MB)
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:1129      Modules in train mode
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:0         Modules in eval mode
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 peft:250] Initializing model parallel
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 megatron_parallel:584]  > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 306774016
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 megatron_parallel:587]  > number of trainable parameters: 8716288 (2.84% of total)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 utils:673] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_megatron_fsdp=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache=False, nccl_ub=False, fsdp_double_buffer=False, outer_dp_sharding_strategy='no_shard', disable_symmetric_registration=False, delay_wgrad_compute=False)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:02 utils:694] Number of buckets for gradient all-reduce / reduce-scatter: 1
ataset_32k/0 [default0]:    Params for bucket 1 (8716288 elements, 8716288 padded size):
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:03 peft:254] Setting up optimizers
ataset_32k/0 [default0]:[NeMo I 2025-11-25 14:09:03 utils:673] Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=None, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='delayed', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.98, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather=False, overlap_param_gather_with_optimizer_step=False, use_megatron_fsdp=False, optimizer_cpu_offload=False, optimizer_offload_fraction=0.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=False, timers=None, config_logger_dir='')
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:10:59 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('global_batch_size', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:[NeMo W 2025-11-25 14:10:59 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('val_loss', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
ataset_32k/0 [default0]: