[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/nemo/qwen3_600m_finetune_34687.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

python "$1"/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
─ Entering Experiment qwen3_600m_lora_finetune_crowd_code_dataset_32k with id… ─
[12:28:57] INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_lora_finetune_crowd_c                       
                    ode_dataset_32k/qwen3_600m_lora_finet                       
                    une_crowd_code_dataset_32k_1764070137                       
                    /qwen3_600m_lora_finetune_crowd_code_                       
                    dataset_32k                                                 
[12:28:57] Launching job                                       experiment.py:744
           qwen3_600m_lora_finetune_crowd_code_dataset_32k for                  
           experiment                                                           
           qwen3_600m_lora_finetune_crowd_code_dataset_32k                      
           INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_lora_finetune_crowd_c                       
                    ode_dataset_32k/qwen3_600m_lora_finet                       
                    une_crowd_code_dataset_32k_1764070137                       
                    /qwen3_600m_lora_finetune_crowd_code_                       
                    dataset_32k                                                 
[12:28:58] INFO     Launched app:                                launcher.py:111
                    local_persistent://nemo_run/qwen3_600m_lora_                
                    finetune_crowd_code_dataset_32k-z2vkw2g5ltjb                
                    qc                                                          
─ Waiting for Experiment qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764… ─

Experiment Status for qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764070137

Task 0: qwen3_600m_lora_finetune_crowd_code_dataset_32k
- Status: RUNNING
- Executor: LocalExecutor
- Job id: qwen3_600m_lora_finetune_crowd_code_dataset_32k-z2vkw2g5ltjbqc
- Local Directory: /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_lora_finetune_crowd_code_dataset_32k/qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764070137/qwen3_600m_lora_finetune_crowd_code_dataset_32k

           INFO     Waiting for job                              launcher.py:130
                    qwen3_600m_lora_finetune_crowd_code_dataset_                
                    32k-z2vkw2g5ltjbqc to finish [log=True]...                  
ataset_32k/0 /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0   import pynvml  # type: ignore[import]
ataset_32k/0 I1125 12:28:59.641000 1100387 torch/distributed/run.py:657] Using nproc_per_node=2.
ataset_32k/0 W1125 12:28:59.642000 1100387 torch/distributed/run.py:774] 
ataset_32k/0 W1125 12:28:59.642000 1100387 torch/distributed/run.py:774] *****************************************
ataset_32k/0 W1125 12:28:59.642000 1100387 torch/distributed/run.py:774] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
ataset_32k/0 W1125 12:28:59.642000 1100387 torch/distributed/run.py:774] *****************************************
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199] Starting elastic_operator with launch configs:
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   entrypoint         : nemo_run.core.runners.fdl_runner
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   min_nodes          : 1
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   max_nodes          : 1
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   nproc_per_node     : 2
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   run_id             : 3845
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   rdzv_backend       : c10d
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   rdzv_endpoint      : localhost:0
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   rdzv_configs       : {'timeout': 900}
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   max_restarts       : 0
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   monitor_interval   : 0.1
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   log_dir            : /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_lora_finetune_crowd_code_dataset_32k/qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764070137/qwen3_600m_lora_finetune_crowd_code_dataset_32k/nemo_run/qwen3_600m_lora_finetune_crowd_code_dataset_32k-z2vkw2g5ltjbqc/torchelastic/qwen3_600m_lora_finetune_crowd_code_dataset_32k
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   metrics_cfg        : {}
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199]   event_log_handler  : null
ataset_32k/0 I1125 12:28:59.645000 1100387 torch/distributed/launcher/api.py:199] 
ataset_32k/0 I1125 12:28:59.651000 1100387 torch/distributed/elastic/agent/server/api.py:869] [default] starting workers for entrypoint: python3
ataset_32k/0 I1125 12:28:59.652000 1100387 torch/distributed/elastic/agent/server/api.py:677] [default] Rendezvous'ing worker group
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523] [default] Rendezvous complete for workers. Result:
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   restart_count=0
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   master_addr=hai005.haicore.berlin
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   master_port=42003
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   group_rank=0
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   group_world_size=1
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   local_ranks=[0, 1]
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   role_ranks=[0, 1]
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   global_ranks=[0, 1]
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   role_world_sizes=[2, 2]
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   global_world_sizes=[2, 2]
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523]   event_log_handler=null
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:523] 
ataset_32k/0 I1125 12:28:59.723000 1100387 torch/distributed/elastic/agent/server/api.py:685] [default] Starting worker group
ataset_32k/0 I1125 12:28:59.724000 1100387 torch/distributed/elastic/agent/server/local_elastic_agent.py:296] use_agent_store: True
ataset_32k/0 I1125 12:28:59.725000 1100387 torch/distributed/elastic/agent/server/local_elastic_agent.py:192] Environment variable 'TORCHELASTIC_ENABLE_FILE_TIMER' not found. Do not start FileTimerServer.
ataset_32k/0 I1125 12:28:59.725000 1100387 torch/distributed/elastic/agent/server/local_elastic_agent.py:236] Environment variable 'TORCHELASTIC_HEALTH_CHECK_PORT' not found. Do not start health check.
ataset_32k/0 [default1]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
ataset_32k/0 [default0]:  import pynvml  # type: ignore[import]
ataset_32k/0 [default1]:  import pynvml  # type: ignore[import]
ataset_32k/0 [default0]:trainer.num_devices: 2
ataset_32k/0 [default0]:trainer.num_nodes: 1
ataset_32k/0 [default0]:trainer.strategy.tensor_model_parallel_size: 1
ataset_32k/0 [default0]:trainer.strategy.pipeline_model_parallel_size: 1
ataset_32k/0 [default0]:trainer.strategy.context_parallel_size: 1
ataset_32k/0 [default0]:data.global_batch_size: 4
ataset_32k/0 [default0]:data.micro_batch_size: 1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:20 api:1251] Disabling try_restore_best_ckpt restoration for adapters
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:20 nemo_logger:162] Experiments will be logged at /fast/project/HFMI_SynergyUnit/tab_model/data/outputs/qwen3_600m_lora_finetune_crowd_code_dataset_32k/2025-11-25_12-29-20
ataset_32k/0 [default0]:GPU available: True (cuda), used: True
ataset_32k/0 [default0]:TPU available: False, using: 0 TPU cores
ataset_32k/0 [default0]:HPU available: False, using: 0 HPUs
ataset_32k/0 [default0]:[NeMo W 2025-11-25 12:29:20 nemo_logger:190] "update_logger_directory" is True. Overwriting tensorboard logger "save_dir" to /fast/project/HFMI_SynergyUnit/tab_model/data/outputs/tb_logs
ataset_32k/0 [default0]:[NeMo W 2025-11-25 12:29:20 nemo_logger:209] The Trainer already contains a ModelCheckpoint callback. This will be overwritten.
ataset_32k/0 [default0]:[NeMo W 2025-11-25 12:29:20 nemo_logger:239] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_strategy:460] Fixing mis-match between ddp-config & mcore-optimizer config
ataset_32k/0 [default0]:You are using a CUDA device ('NVIDIA H100 80GB HBM3') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:449] Rank 0 has data parallel group : [0, 1]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:455] Rank 0 has combined group of data parallel and context parallel : [0, 1]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:460] All data parallel group ranks with context parallel combined: [[0, 1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:463] Ranks 0 has data parallel rank: 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:471] Rank 0 has context parallel group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:474] All context parallel group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:475] Ranks 0 has context parallel rank: 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:482] Rank 0 has model parallel group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:483] All model parallel group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:492] Rank 0 has tensor model parallel group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:496] All tensor model parallel group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:497] Rank 0 has tensor model parallel rank: 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:532] Rank 0 has pipeline model parallel group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:544] Rank 0 has embedding group: [0]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:550] All pipeline model parallel group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:551] Rank 0 has pipeline model parallel rank 0
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:552] All embedding group ranks: [[0], [1]]
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:21 megatron_init:553] Rank 0 has embedding rank: 0
ataset_32k/0 [default0]:[W1125 12:29:21.075648823 ProcessGroupNCCL.cpp:981] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
ataset_32k/0 [default0]:----------------------------------------------------------------------------------------------------
ataset_32k/0 [default0]:distributed_backend=nccl
ataset_32k/0 [default0]:All distributed processes registered. Starting with 2 processes
ataset_32k/0 [default0]:----------------------------------------------------------------------------------------------------
ataset_32k/0 [default0]:
ataset_32k/0 [default1]:trainer.num_devices: 2
ataset_32k/0 [default1]:trainer.num_nodes: 1
ataset_32k/0 [default1]:trainer.strategy.tensor_model_parallel_size: 1
ataset_32k/0 [default1]:trainer.strategy.pipeline_model_parallel_size: 1
ataset_32k/0 [default1]:trainer.strategy.context_parallel_size: 1
ataset_32k/0 [default1]:data.global_batch_size: 4
ataset_32k/0 [default1]:data.micro_batch_size: 1
ataset_32k/0 [default1]:[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default1]:[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default1]:[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default1]:[W1125 12:29:26.093129119 ProcessGroupNCCL.cpp:981] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
ataset_32k/0 [default1]:[Gloo] Rank 1 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default0]:[Gloo] Rank 0 is connected to 1 peer ranks. Expected number of connected peer ranks is : 1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:27 model_transform:67] Setting up ModelTransform for stage: fit
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:27 model_transform:70] Found model_transform attribute on pl_module
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:27 model_transform:73] Set model_transform to: <function _call_counter.<locals>.wrapper at 0x7ff0e435b760>
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:27 base:373] Use preset vocab_size: 151936, original vocab_size: 151669, dummy tokens: 267.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:27 megatron_strategy:472] Copying Trainer's 'max_steps' (1000) to LR scheduler's 'max_steps'.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:27 num_microbatches_calculator:228] setting number of microbatches to constant 2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:27 megatron_strategy:1045] Doing selective restore from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:27 pl:411] Loaded sharded_state_dict_metadata from checkpoint: {'distrib_optim_sharding_type': 'fully_sharded_model_space'}
ataset_32k/0 [default1]:LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1]
ataset_32k/0 [default0]:LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]
ataset_32k/0 [default0]:[NeMo W 2025-11-25 12:29:27 serialization:184] DEPRECATED: Passing 'checkpoint_dir' as a Path object in load_common_state_dict will no longer be supported in a future release. Please pass it as a string instead.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:27 pl:275] Using <megatron.core.dist_checkpointing.strategies.fully_parallel.FullyParallelLoadStrategyWrapper object at 0x7ff10d7a28c0> dist-ckpt load strategy.
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 pl:297] Global Checkpoint Load : Rank : 0 : Start time : 1764070167.608s : Time spent in load_checkpoint: 1.010s
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 megatron_strategy:1050] Restoring model weights from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 megatron_strategy:1058] Finished restoring from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True), cleaning up.
ataset_32k/0 [default0]:
ataset_32k/0 [default0]:  | Name   | Type     | Params | Mode 
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:0 | module | GPTModel | 596 M  | train
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:596 M     Trainable params
ataset_32k/0 [default0]:0         Non-trainable params
ataset_32k/0 [default0]:596 M     Total params
ataset_32k/0 [default0]:2,384.200 Total estimated model params size (MB)
ataset_32k/0 [default0]:569       Modules in train mode
ataset_32k/0 [default0]:0         Modules in eval mode
ataset_32k/0 [default0]:[NeMo W 2025-11-25 12:29:28 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=4` in the `DataLoader` to improve performance.
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.0.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.0.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.0.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.0.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.1.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.1.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.1.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.1.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.2.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.2.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.2.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.2.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.3.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.3.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.3.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.3.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.4.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.4.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.4.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.4.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.5.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.5.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.5.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.5.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.6.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.6.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.6.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.6.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.7.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.7.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.7.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.7.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.8.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.8.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.8.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.8.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.9.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.9.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.9.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.9.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.10.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.10.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.10.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.10.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:28 lora:468] Adding lora to: module.decoder.layers.11.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.11.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.11.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.11.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.12.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.12.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.12.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.12.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.13.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.13.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.13.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.13.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.14.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.14.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.14.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.14.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.15.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.15.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.15.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.15.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.16.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.16.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.16.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.16.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.17.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.17.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.17.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.17.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.18.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.18.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.18.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.18.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.19.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.19.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.19.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.19.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.20.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.20.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.20.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.20.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.21.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.21.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.21.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.21.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.22.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.22.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.22.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.22.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.23.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.23.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.23.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.23.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.24.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.24.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.24.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.24.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.25.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.25.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.25.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.25.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.26.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.26.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.26.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.26.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.27.self_attention.linear_proj
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.27.self_attention.linear_qkv
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.27.mlp.linear_fc1
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 lora:468] Adding lora to: module.decoder.layers.27.mlp.linear_fc2
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 model_transform:97] After applying model_transform:
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:  | Name   | Type     | Params | Mode 
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:0 | module | GPTModel | 613 M  | train
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:--------------------------------------------
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:17.4 M    Trainable params
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:596 M     Non-trainable params
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:613 M     Total params
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:2,453.930 Total estimated model params size (MB)
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:1129      Modules in train mode
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:0         Modules in eval mode
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 peft:250] Initializing model parallel
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 megatron_parallel:584]  > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 613482496
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 megatron_parallel:587]  > number of trainable parameters: 17432576 (2.84% of total)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 utils:673] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_megatron_fsdp=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache=False, nccl_ub=False, fsdp_double_buffer=False, outer_dp_sharding_strategy='no_shard', disable_symmetric_registration=False, delay_wgrad_compute=False)
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 utils:694] Number of buckets for gradient all-reduce / reduce-scatter: 1
ataset_32k/0 [default0]:    Params for bucket 1 (17432576 elements, 17432576 padded size):
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.adapter.linear_out.weight
ataset_32k/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.adapter.linear_in.weight
ataset_32k/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.adapter.linear_out.weight
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 peft:254] Setting up optimizers
ataset_32k/0 [default0]:[NeMo I 2025-11-25 12:29:29 utils:673] Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=None, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='delayed', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.98, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather=False, overlap_param_gather_with_optimizer_step=False, use_megatron_fsdp=False, optimizer_cpu_offload=False, optimizer_offload_fraction=0.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=False, timers=None, config_logger_dir='')
ataset_32k/0 [default0]:[NeMo W 2025-11-25 12:30:22 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('global_batch_size', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:[NeMo W 2025-11-25 12:30:22 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('val_loss', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
ataset_32k/0 [default0]:    
ataset_32k/0 [default0]:Sanity checking Validation: iteration 1/1
ataset_32k/0 [default0]:[NeMo W 2025-11-25 12:30:22 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=4` in the `DataLoader` to improve performance.
ataset_32k/0 [default0]:    
ataset_32k/0 [default1]:[rank1]: Traceback (most recent call last):
ataset_32k/0 [default1]:[rank1]:   File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main
ataset_32k/0 [default1]:[rank1]:     return _run_code(code, main_globals, None,
ataset_32k/0 [default1]:[rank1]:   File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code
ataset_32k/0 [default1]:[rank1]:     exec(code, run_globals)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 72, in <module>
ataset_32k/0 [default1]:[rank1]:     fdl_runner_app()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 327, in __call__
ataset_32k/0 [default1]:[rank1]:     raise e
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 310, in __call__
ataset_32k/0 [default1]:[rank1]:     return get_command(self)(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1485, in __call__
ataset_32k/0 [default1]:[rank1]:     return self.main(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 719, in main
ataset_32k/0 [default1]:[rank1]:     return _main(
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 192, in _main
ataset_32k/0 [default1]:[rank1]:     rv = self.invoke(ctx)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1269, in invoke
ataset_32k/0 [default1]:[rank1]:     return ctx.invoke(self.callback, **ctx.params)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 824, in invoke
ataset_32k/0 [default1]:[rank1]:     return callback(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 691, in wrapper
ataset_32k/0 [default1]:[rank1]:     return callback(**use_params)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 68, in fdl_direct_run
ataset_32k/0 [default0]:[rank0]: Traceback (most recent call last):
ataset_32k/0 [default1]:[rank1]:     fdl_fn()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/api.py", line 237, in finetune
ataset_32k/0 [default0]:[rank0]:   File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main
ataset_32k/0 [default0]:[rank0]:     return _run_code(code, main_globals, None,
ataset_32k/0 [default1]:[rank1]:     return train(
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/api.py", line 136, in train
ataset_32k/0 [default0]:[rank0]:   File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code
ataset_32k/0 [default0]:[rank0]:     exec(code, run_globals)
ataset_32k/0 [default1]:[rank1]:     trainer.fit(model, data)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 538, in fit
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 72, in <module>
ataset_32k/0 [default0]:[rank0]:     fdl_runner_app()
ataset_32k/0 [default1]:[rank1]:     call._call_and_handle_interrupt(
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 46, in _call_and_handle_interrupt
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 327, in __call__
ataset_32k/0 [default0]:[rank0]:     raise e
ataset_32k/0 [default1]:[rank1]:     return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 310, in __call__
ataset_32k/0 [default0]:[rank0]:     return get_command(self)(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:     return function(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 574, in _fit_impl
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1485, in __call__
ataset_32k/0 [default0]:[rank0]:     return self.main(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:     self._run(model, ckpt_path=ckpt_path)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 981, in _run
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 719, in main
ataset_32k/0 [default0]:[rank0]:     return _main(
ataset_32k/0 [default1]:[rank1]:     results = self._run_stage()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1025, in _run_stage
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 192, in _main
ataset_32k/0 [default0]:[rank0]:     rv = self.invoke(ctx)
ataset_32k/0 [default1]:[rank1]:     self.fit_loop.run()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 205, in run
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1269, in invoke
ataset_32k/0 [default0]:[rank0]:     return ctx.invoke(self.callback, **ctx.params)
ataset_32k/0 [default1]:[rank1]:     self.advance()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 363, in advance
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 824, in invoke
ataset_32k/0 [default0]:[rank0]:     return callback(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:     self.epoch_loop.run(self._data_fetcher)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 140, in run
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 691, in wrapper
ataset_32k/0 [default0]:[rank0]:     return callback(**use_params)
ataset_32k/0 [default1]:[rank1]:     self.advance(data_fetcher)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/pytorch/trainer.py", line 47, in advance
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 68, in fdl_direct_run
ataset_32k/0 [default0]:[rank0]:     fdl_fn()
ataset_32k/0 [default1]:[rank1]:     super().advance(data_fetcher)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 250, in advance
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/api.py", line 237, in finetune
ataset_32k/0 [default0]:[rank0]:     return train(
ataset_32k/0 [default1]:[rank1]:     batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 190, in run
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/api.py", line 136, in train
ataset_32k/0 [default0]:[rank0]:     trainer.fit(model, data)
ataset_32k/0 [default1]:[rank1]:     self._optimizer_step(batch_idx, closure)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 268, in _optimizer_step
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 538, in fit
ataset_32k/0 [default0]:[rank0]:     call._call_and_handle_interrupt(
ataset_32k/0 [default1]:[rank1]:     call._call_lightning_module_hook(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 46, in _call_and_handle_interrupt
ataset_32k/0 [default0]:[rank0]:     return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 167, in _call_lightning_module_hook
ataset_32k/0 [default1]:[rank1]:     output = fn(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
ataset_32k/0 [default0]:[rank0]:     return function(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/core/module.py", line 1306, in optimizer_step
ataset_32k/0 [default1]:[rank1]:     optimizer.step(closure=optimizer_closure)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 574, in _fit_impl
ataset_32k/0 [default0]:[rank0]:     self._run(model, ckpt_path=ckpt_path)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/core/optimizer.py", line 153, in step
ataset_32k/0 [default1]:[rank1]:     step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 981, in _run
ataset_32k/0 [default0]:[rank0]:     results = self._run_stage()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/pytorch/strategies/megatron_strategy.py", line 811, in optimizer_step
ataset_32k/0 [default1]:[rank1]:     optimizer_output = super().optimizer_step(optimizer, closure, model, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1025, in _run_stage
ataset_32k/0 [default0]:[rank0]:     self.fit_loop.run()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/strategies/ddp.py", line 270, in optimizer_step
ataset_32k/0 [default1]:[rank1]:     optimizer_output = super().optimizer_step(optimizer, closure, model, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 205, in run
ataset_32k/0 [default0]:[rank0]:     self.advance()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/strategies/strategy.py", line 238, in optimizer_step
ataset_32k/0 [default1]:[rank1]:     return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 363, in advance
ataset_32k/0 [default0]:[rank0]:     self.epoch_loop.run(self._data_fetcher)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/precision.py", line 122, in optimizer_step
ataset_32k/0 [default1]:[rank1]:     return optimizer.step(closure=closure, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 140, in run
ataset_32k/0 [default0]:[rank0]:     self.advance(data_fetcher)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/optim/lr_scheduler.py", line 133, in wrapper
ataset_32k/0 [default1]:[rank1]:     return func.__get__(opt, opt.__class__)(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/pytorch/trainer.py", line 47, in advance
ataset_32k/0 [default0]:[rank0]:     super().advance(data_fetcher)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/core/optim/mcore_optim.py", line 143, in step
ataset_32k/0 [default1]:[rank1]:     loss = closure()
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 250, in advance
ataset_32k/0 [default0]:[rank0]:     batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/precision.py", line 108, in _wrap_closure
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 190, in run
ataset_32k/0 [default0]:[rank0]:     self._optimizer_step(batch_idx, closure)
ataset_32k/0 [default1]:[rank1]:     closure_result = closure()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 144, in __call__
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 268, in _optimizer_step
ataset_32k/0 [default0]:[rank0]:     call._call_lightning_module_hook(
ataset_32k/0 [default1]:[rank1]:     self._result = self.closure(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 167, in _call_lightning_module_hook
ataset_32k/0 [default0]:[rank0]:     output = fn(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
ataset_32k/0 [default1]:[rank1]:     return func(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/core/module.py", line 1306, in optimizer_step
ataset_32k/0 [default0]:[rank0]:     optimizer.step(closure=optimizer_closure)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 129, in closure
ataset_32k/0 [default1]:[rank1]:     step_output = self._step_fn()
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/core/optimizer.py", line 153, in step
ataset_32k/0 [default0]:[rank0]:     step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 317, in _training_step
ataset_32k/0 [default1]:[rank1]:     training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/pytorch/strategies/megatron_strategy.py", line 811, in optimizer_step
ataset_32k/0 [default0]:[rank0]:     optimizer_output = super().optimizer_step(optimizer, closure, model, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 319, in _call_strategy_hook
ataset_32k/0 [default1]:[rank1]:     output = fn(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/strategies/ddp.py", line 270, in optimizer_step
ataset_32k/0 [default0]:[rank0]:     optimizer_output = super().optimizer_step(optimizer, closure, model, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/pytorch/strategies/megatron_strategy.py", line 743, in training_step
ataset_32k/0 [default1]:[rank1]:     out = self.model.training_step(dataloader_iter, *args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/strategies/strategy.py", line 238, in optimizer_step
ataset_32k/0 [default0]:[rank0]:     return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 346, in training_step
ataset_32k/0 [default1]:[rank1]:     return self._step(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/precision.py", line 122, in optimizer_step
ataset_32k/0 [default0]:[rank0]:     return optimizer.step(closure=closure, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 458, in _step
ataset_32k/0 [default1]:[rank1]:     return self.forward(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/optim/lr_scheduler.py", line 133, in wrapper
ataset_32k/0 [default0]:[rank0]:     return func.__get__(opt, opt.__class__)(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 308, in forward
ataset_32k/0 [default1]:[rank1]:     microbatch_outputs = step()
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/core/optim/mcore_optim.py", line 143, in step
ataset_32k/0 [default0]:[rank0]:     loss = closure()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 1248, in __call__
ataset_32k/0 [default1]:[rank1]:     return self.forward_backward_func(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/precision.py", line 108, in _wrap_closure
ataset_32k/0 [default0]:[rank0]:     closure_result = closure()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/pipeline_parallel/schedules.py", line 600, in forward_backward_no_pipelining
ataset_32k/0 [default1]:[rank1]:     output_tensor, num_tokens = forward_step(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 144, in __call__
ataset_32k/0 [default0]:[rank0]:     self._result = self.closure(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/pipeline_parallel/schedules.py", line 400, in forward_step
ataset_32k/0 [default1]:[rank1]:     output_tensor, loss_func = forward_step_func(data_iterator, model)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
ataset_32k/0 [default0]:[rank0]:     return func(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 523, in wrapped_forward_step_func
ataset_32k/0 [default1]:[rank1]:     output_tensor = _forward_step(model, batch)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 129, in closure
ataset_32k/0 [default0]:[rank0]:     step_output = self._step_fn()
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 858, in wrapped
ataset_32k/0 [default1]:[rank1]:     return attr(*args)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 317, in _training_step
ataset_32k/0 [default0]:[rank0]:     training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/model/base.py", line 679, in training_step
ataset_32k/0 [default1]:[rank1]:     return self.forward_step(batch)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 319, in _call_strategy_hook
ataset_32k/0 [default0]:[rank0]:     output = fn(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/model/base.py", line 666, in forward_step
ataset_32k/0 [default1]:[rank1]:     return self.config.forward_step_fn(self, batch)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/pytorch/strategies/megatron_strategy.py", line 743, in training_step
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/model/base.py", line 143, in gpt_forward_step
ataset_32k/0 [default1]:[rank1]:     return model(**forward_args)
ataset_32k/0 [default0]:[rank0]:     out = self.model.training_step(dataloader_iter, *args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 346, in training_step
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
ataset_32k/0 [default0]:[rank0]:     return self._step(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 458, in _step
ataset_32k/0 [default1]:[rank1]:     return self._call_impl(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
ataset_32k/0 [default0]:[rank0]:     return self.forward(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 308, in forward
ataset_32k/0 [default1]:[rank1]:     return forward_call(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/model/base.py", line 634, in forward
ataset_32k/0 [default0]:[rank0]:     microbatch_outputs = step()
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 1248, in __call__
ataset_32k/0 [default1]:[rank1]:     output_tensor = self.module(
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
ataset_32k/0 [default0]:[rank0]:     return self.forward_backward_func(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/pipeline_parallel/schedules.py", line 600, in forward_backward_no_pipelining
ataset_32k/0 [default1]:[rank1]:     return self._call_impl(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
ataset_32k/0 [default0]:[rank0]:     output_tensor, num_tokens = forward_step(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/pipeline_parallel/schedules.py", line 400, in forward_step
ataset_32k/0 [default1]:[rank1]:     return forward_call(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/distributed/data_parallel_base.py", line 22, in forward
ataset_32k/0 [default0]:[rank0]:     output_tensor, loss_func = forward_step_func(data_iterator, model)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 523, in wrapped_forward_step_func
ataset_32k/0 [default1]:[rank1]:     return self.module(*inputs, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
ataset_32k/0 [default0]:[rank0]:     output_tensor = _forward_step(model, batch)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/megatron_parallel.py", line 858, in wrapped
ataset_32k/0 [default1]:[rank1]:     return self._call_impl(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
ataset_32k/0 [default0]:[rank0]:     return attr(*args)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/model/base.py", line 679, in training_step
ataset_32k/0 [default1]:[rank1]:     return forward_call(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/transformer/module.py", line 237, in forward
ataset_32k/0 [default0]:[rank0]:     return self.forward_step(batch)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/model/base.py", line 666, in forward_step
ataset_32k/0 [default1]:[rank1]:     outputs = self.module(*inputs, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
ataset_32k/0 [default0]:[rank0]:     return self.config.forward_step_fn(self, batch)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/model/base.py", line 143, in gpt_forward_step
ataset_32k/0 [default1]:[rank1]:     return self._call_impl(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
ataset_32k/0 [default0]:[rank0]:     return model(**forward_args)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
ataset_32k/0 [default1]:[rank1]:     return forward_call(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/models/gpt/gpt_model.py", line 395, in forward
ataset_32k/0 [default0]:[rank0]:     return self._call_impl(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
ataset_32k/0 [default1]:[rank1]:     return self._postprocess(
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/models/gpt/gpt_model.py", line 562, in _postprocess
ataset_32k/0 [default0]:[rank0]:     return forward_call(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/gpt/model/base.py", line 634, in forward
ataset_32k/0 [default1]:[rank1]:     loss = self.compute_language_model_loss(labels, logits)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/models/common/language_module/language_module.py", line 157, in compute_language_model_loss
ataset_32k/0 [default0]:[rank0]:     output_tensor = self.module(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
ataset_32k/0 [default1]:[rank1]:     loss = fused_vocab_parallel_cross_entropy(logits, labels, self.model_comm_pgs.tp)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/fusions/fused_cross_entropy.py", line 148, in fused_vocab_parallel_cross_entropy
ataset_32k/0 [default0]:[rank0]:     return self._call_impl(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
ataset_32k/0 [default1]:[rank1]:     return _VocabParallelCrossEntropy.apply(vocab_parallel_logits, target, tp_group)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/autograd/function.py", line 576, in apply
ataset_32k/0 [default0]:[rank0]:     return forward_call(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/distributed/data_parallel_base.py", line 22, in forward
ataset_32k/0 [default1]:[rank1]:     return super().apply(*args, **kwargs)  # type: ignore[misc]
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/fusions/fused_cross_entropy.py", line 93, in forward
ataset_32k/0 [default0]:[rank0]:     return self.module(*inputs, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
ataset_32k/0 [default1]:[rank1]:     vocab_parallel_logits, logits_max = calculate_logits_max(vocab_parallel_logits)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 736, in compile_wrapper
ataset_32k/0 [default0]:[rank0]:     return self._call_impl(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
ataset_32k/0 [default1]:[rank1]:     return fn(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/fusions/fused_cross_entropy.py", line 12, in calculate_logits_max
ataset_32k/0 [default0]:[rank0]:     return forward_call(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/transformer/module.py", line 237, in forward
ataset_32k/0 [default1]:[rank1]:     @jit_fuser
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 929, in _fn
ataset_32k/0 [default0]:[rank0]:     outputs = self.module(*inputs, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
ataset_32k/0 [default1]:[rank1]:     return fn(*args, **kwargs)
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1241, in forward
ataset_32k/0 [default0]:[rank0]:     return self._call_impl(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
ataset_32k/0 [default1]:[rank1]:     return compiled_fn(full_args)
ataset_32k/0 [default0]:[rank0]:     return forward_call(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/models/gpt/gpt_model.py", line 395, in forward
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 384, in runtime_wrapper
ataset_32k/0 [default1]:[rank1]:     all_outs = call_func_at_runtime_with_args(
ataset_32k/0 [default0]:[rank0]:     return self._postprocess(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/models/gpt/gpt_model.py", line 562, in _postprocess
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args
ataset_32k/0 [default1]:[rank1]:     out = normalize_as_list(f(args))
ataset_32k/0 [default0]:[rank0]:     loss = self.compute_language_model_loss(labels, logits)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/models/common/language_module/language_module.py", line 157, in compute_language_model_loss
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 750, in inner_fn
ataset_32k/0 [default1]:[rank1]:     outs = compiled_fn(args)
ataset_32k/0 [default0]:[rank0]:     loss = fused_vocab_parallel_cross_entropy(logits, labels, self.model_comm_pgs.tp)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/fusions/fused_cross_entropy.py", line 148, in fused_vocab_parallel_cross_entropy
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 556, in wrapper
ataset_32k/0 [default1]:[rank1]:     return compiled_fn(runtime_args)
ataset_32k/0 [default0]:[rank0]:     return _VocabParallelCrossEntropy.apply(vocab_parallel_logits, target, tp_group)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/autograd/function.py", line 576, in apply
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 584, in __call__
ataset_32k/0 [default1]:[rank1]:     return self.current_callable(inputs)
ataset_32k/0 [default0]:[rank0]:     return super().apply(*args, **kwargs)  # type: ignore[misc]
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/fusions/fused_cross_entropy.py", line 93, in forward
ataset_32k/0 [default1]:[rank1]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_inductor/utils.py", line 2716, in run
ataset_32k/0 [default1]:[rank1]:     out = model(new_inputs)
ataset_32k/0 [default0]:[rank0]:     vocab_parallel_logits, logits_max = calculate_logits_max(vocab_parallel_logits)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 736, in compile_wrapper
ataset_32k/0 [default1]:[rank1]:   File "/var/tmp/torchinductor_mihir.mahajan/u6/cu6tksnqbxs6c2eqyyq7nwisjr6nbfb3b2gwn4aqod3gbod7quqg.py", line 99, in call
ataset_32k/0 [default1]:[rank1]:     buf0 = empty_strided_cuda((s10, 1, 151936), (151936, 151936, 1), torch.float32)
ataset_32k/0 [default0]:[rank0]:     return fn(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/megatron/core/fusions/fused_cross_entropy.py", line 12, in calculate_logits_max
ataset_32k/0 [default1]:[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 18.55 GiB. GPU 1 has a total capacity of 79.19 GiB of which 13.09 GiB is free. Including non-PyTorch memory, this process has 65.99 GiB memory in use. Of the allocated memory 57.72 GiB is allocated by PyTorch, and 5.10 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
ataset_32k/0 [default0]:[rank0]:     @jit_fuser
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 929, in _fn
ataset_32k/0 [default0]:[rank0]:     return fn(*args, **kwargs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1241, in forward
ataset_32k/0 [default0]:[rank0]:     return compiled_fn(full_args)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 384, in runtime_wrapper
ataset_32k/0 [default0]:[rank0]:     all_outs = call_func_at_runtime_with_args(
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 126, in call_func_at_runtime_with_args
ataset_32k/0 [default0]:[rank0]:     out = normalize_as_list(f(args))
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 750, in inner_fn
ataset_32k/0 [default0]:[rank0]:     outs = compiled_fn(args)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 556, in wrapper
ataset_32k/0 [default0]:[rank0]:     return compiled_fn(runtime_args)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 584, in __call__
ataset_32k/0 [default0]:[rank0]:     return self.current_callable(inputs)
ataset_32k/0 [default0]:[rank0]:   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_inductor/utils.py", line 2716, in run
ataset_32k/0 [default0]:[rank0]:     out = model(new_inputs)
ataset_32k/0 [default0]:[rank0]:   File "/var/tmp/torchinductor_mihir.mahajan/yb/cyb57exsxglzogtek72lqpgvh7nyqgvr67fp73yo7ti4zpzl3wa3.py", line 99, in call
ataset_32k/0 [default0]:[rank0]:     buf0 = empty_strided_cuda((32768, 1, 151936), (151936, 151936, 1), torch.float32)
ataset_32k/0 [default0]:[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 18.55 GiB. GPU 0 has a total capacity of 79.19 GiB of which 6.48 GiB is free. Including non-PyTorch memory, this process has 72.63 GiB memory in use. Of the allocated memory 57.71 GiB is allocated by PyTorch, and 11.77 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
ataset_32k/0 W1125 12:30:39.425000 1100387 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 1100407 closing signal SIGTERM
[12:30:44] INFO     Job                                          launcher.py:160
                    qwen3_600m_lora_finetune_crowd_code_dataset_                
                    32k-z2vkw2g5ltjbqc finished: FAILED                         
                                                                                
# The experiment was run with the following tasks: ['qwen3_600m_lora_finetune_cr
# You can inspect and reconstruct this experiment at a later point in time using
experiment = run.Experiment.from_id("qwen3_600m_lora_finetune_crowd_code_dataset
experiment.status() # Gets the overall status                                   
experiment.logs("qwen3_600m_lora_finetune_crowd_code_dataset_32k") # Gets the lo
experiment.cancel("qwen3_600m_lora_finetune_crowd_code_dataset_32k") # Cancels t
                                                                                
                                                                                
# You can inspect this experiment at a later point in time using the CLI as well
nemo experiment status qwen3_600m_lora_finetune_crowd_code_dataset_32k_176407013
nemo experiment logs qwen3_600m_lora_finetune_crowd_code_dataset_32k_1764070137 
nemo experiment cancel qwen3_600m_lora_finetune_crowd_code_dataset_32k_176407013
                                                                                
ataset_32k/0 E1125 12:30:43.207000 1100387 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 0 (pid: 1100406) of binary: /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/bin/python3
ataset_32k/0 I1125 12:30:43.213000 1100387 torch/distributed/elastic/multiprocessing/errors/__init__.py:370] ('local_rank %s FAILED with no error file. Decorate your entrypoint fn with @record for traceback info. See: https://pytorch.org/docs/stable/elastic/errors.html', 0)
ataset_32k/0 Traceback (most recent call last):
ataset_32k/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/bin/torchrun", line 10, in <module>
ataset_32k/0     sys.exit(main())
ataset_32k/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
ataset_32k/0     return f(*args, **kwargs)
ataset_32k/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main
ataset_32k/0     run(args)
ataset_32k/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run
ataset_32k/0     elastic_launch(
ataset_32k/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__
ataset_32k/0     return launch_agent(self._config, self._entrypoint, list(args))
ataset_32k/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent
ataset_32k/0     raise ChildFailedError(
ataset_32k/0 torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
ataset_32k/0 ============================================================
ataset_32k/0 nemo_run.core.runners.fdl_runner FAILED
ataset_32k/0 ------------------------------------------------------------
ataset_32k/0 Failures:
ataset_32k/0   <NO_OTHER_FAILURES>
ataset_32k/0 ------------------------------------------------------------
ataset_32k/0 Root Cause (first observed failure):
ataset_32k/0 [0]:
ataset_32k/0   time      : 2025-11-25_12:30:39
ataset_32k/0   host      : hai005.haicore.berlin
ataset_32k/0   rank      : 0 (local_rank: 0)
ataset_32k/0   exitcode  : 1 (pid: 1100406)
ataset_32k/0   error_file: <N/A>
ataset_32k/0   traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
ataset_32k/0 ============================================================