[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/nemo/qwen3_600m_finetune_34700.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

python "$1"/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
─ Entering Experiment qwen3_600m_lora_4k_tokens_act_ckpt with id: qwen3_600m_… ─
[14:56:24] INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_lora_4k_tokens_act_ck                       
                    pt/qwen3_600m_lora_4k_tokens_act_ckpt                       
                    _1764078984/qwen3_600m_lora_4k_tokens                       
                    _act_ckpt                                                   
[14:56:24] Launching job qwen3_600m_lora_4k_tokens_act_ckpt    experiment.py:744
           for experiment qwen3_600m_lora_4k_tokens_act_ckpt                    
           INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_lora_4k_tokens_act_ck                       
                    pt/qwen3_600m_lora_4k_tokens_act_ckpt                       
                    _1764078984/qwen3_600m_lora_4k_tokens                       
                    _act_ckpt                                                   
           INFO     Launched app:                                launcher.py:111
                    local_persistent://nemo_run/qwen3_600m_lora_                
                    4k_tokens_act_ckpt-xzg17zc4ztp6mc                           
─ Waiting for Experiment qwen3_600m_lora_4k_tokens_act_ckpt_1764078984 to fin… ─

Experiment Status for qwen3_600m_lora_4k_tokens_act_ckpt_1764078984

Task 0: qwen3_600m_lora_4k_tokens_act_ckpt
- Status: RUNNING
- Executor: LocalExecutor
- Job id: qwen3_600m_lora_4k_tokens_act_ckpt-xzg17zc4ztp6mc
- Local Directory: /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_lora_4k_tokens_act_ckpt/qwen3_600m_lora_4k_tokens_act_ckpt_1764078984/qwen3_600m_lora_4k_tokens_act_ckpt

[14:56:25] INFO     Waiting for job                              launcher.py:130
                    qwen3_600m_lora_4k_tokens_act_ckpt-xzg17zc4z                
                    tp6mc to finish [log=True]...                               
s_act_ckpt/0 /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
s_act_ckpt/0   import pynvml  # type: ignore[import]
s_act_ckpt/0 I1125 14:56:27.379000 1317503 torch/distributed/run.py:657] Using nproc_per_node=1.
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199] Starting elastic_operator with launch configs:
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   entrypoint         : nemo_run.core.runners.fdl_runner
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   min_nodes          : 1
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   max_nodes          : 1
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   nproc_per_node     : 1
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   run_id             : 5833
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   rdzv_backend       : c10d
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   rdzv_endpoint      : localhost:0
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   rdzv_configs       : {'timeout': 900}
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   max_restarts       : 0
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   monitor_interval   : 0.1
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   log_dir            : /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_lora_4k_tokens_act_ckpt/qwen3_600m_lora_4k_tokens_act_ckpt_1764078984/qwen3_600m_lora_4k_tokens_act_ckpt/nemo_run/qwen3_600m_lora_4k_tokens_act_ckpt-xzg17zc4ztp6mc/torchelastic/qwen3_600m_lora_4k_tokens_act_ckpt
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   metrics_cfg        : {}
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199]   event_log_handler  : null
s_act_ckpt/0 I1125 14:56:27.382000 1317503 torch/distributed/launcher/api.py:199] 
s_act_ckpt/0 I1125 14:56:27.389000 1317503 torch/distributed/elastic/agent/server/api.py:869] [default] starting workers for entrypoint: python3
s_act_ckpt/0 I1125 14:56:27.389000 1317503 torch/distributed/elastic/agent/server/api.py:677] [default] Rendezvous'ing worker group
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523] [default] Rendezvous complete for workers. Result:
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   restart_count=0
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   master_addr=hai001.haicore.berlin
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   master_port=38013
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   group_rank=0
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   group_world_size=1
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   local_ranks=[0]
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   role_ranks=[0]
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   global_ranks=[0]
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   role_world_sizes=[1]
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   global_world_sizes=[1]
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523]   event_log_handler=null
s_act_ckpt/0 I1125 14:56:27.482000 1317503 torch/distributed/elastic/agent/server/api.py:523] 
s_act_ckpt/0 I1125 14:56:27.483000 1317503 torch/distributed/elastic/agent/server/api.py:685] [default] Starting worker group
s_act_ckpt/0 I1125 14:56:27.484000 1317503 torch/distributed/elastic/agent/server/local_elastic_agent.py:296] use_agent_store: True
s_act_ckpt/0 I1125 14:56:27.484000 1317503 torch/distributed/elastic/agent/server/local_elastic_agent.py:192] Environment variable 'TORCHELASTIC_ENABLE_FILE_TIMER' not found. Do not start FileTimerServer.
s_act_ckpt/0 I1125 14:56:27.485000 1317503 torch/distributed/elastic/agent/server/local_elastic_agent.py:236] Environment variable 'TORCHELASTIC_HEALTH_CHECK_PORT' not found. Do not start health check.
s_act_ckpt/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
s_act_ckpt/0 [default0]:  import pynvml  # type: ignore[import]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 api:1251] Disabling try_restore_best_ckpt restoration for adapters
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 nemo_logger:162] Experiments will be logged at /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51
s_act_ckpt/0 [default0]:GPU available: True (cuda), used: True
s_act_ckpt/0 [default0]:TPU available: False, using: 0 TPU cores
s_act_ckpt/0 [default0]:HPU available: False, using: 0 HPUs
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 14:56:51 nemo_logger:190] "update_logger_directory" is True. Overwriting tensorboard logger "save_dir" to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/tb_logs
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 14:56:51 nemo_logger:209] The Trainer already contains a ModelCheckpoint callback. This will be overwritten.
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 14:56:51 nemo_logger:239] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_strategy:460] Fixing mis-match between ddp-config & mcore-optimizer config
s_act_ckpt/0 [default0]:You are using a CUDA device ('NVIDIA H100 80GB HBM3') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:449] Rank 0 has data parallel group : [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:455] Rank 0 has combined group of data parallel and context parallel : [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:460] All data parallel group ranks with context parallel combined: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:463] Ranks 0 has data parallel rank: 0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:471] Rank 0 has context parallel group: [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:474] All context parallel group ranks: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:475] Ranks 0 has context parallel rank: 0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:482] Rank 0 has model parallel group: [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:483] All model parallel group ranks: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:492] Rank 0 has tensor model parallel group: [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:496] All tensor model parallel group ranks: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:497] Rank 0 has tensor model parallel rank: 0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:532] Rank 0 has pipeline model parallel group: [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:544] Rank 0 has embedding group: [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:550] All pipeline model parallel group ranks: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:551] Rank 0 has pipeline model parallel rank 0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:552] All embedding group ranks: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:51 megatron_init:553] Rank 0 has embedding rank: 0
s_act_ckpt/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : [default0]:0[default0]:
s_act_ckpt/0 [default0]:[W1125 14:56:51.438427720 ProcessGroupNCCL.cpp:981] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
s_act_ckpt/0 [default0]:----------------------------------------------------------------------------------------------------
s_act_ckpt/0 [default0]:distributed_backend=nccl
s_act_ckpt/0 [default0]:All distributed processes registered. Starting with 1 processes
s_act_ckpt/0 [default0]:----------------------------------------------------------------------------------------------------
s_act_ckpt/0 [default0]:
s_act_ckpt/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
s_act_ckpt/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
s_act_ckpt/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:52 model_transform:67] Setting up ModelTransform for stage: fit
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:52 model_transform:70] Found model_transform attribute on pl_module
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:52 model_transform:73] Set model_transform to: <function _call_counter.<locals>.wrapper at 0x7f20a00bf7f0>
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:52 base:373] Use preset vocab_size: 151936, original vocab_size: 151669, dummy tokens: 267.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:52 megatron_strategy:472] Copying Trainer's 'max_steps' (1000) to LR scheduler's 'max_steps'.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:52 num_microbatches_calculator:228] setting number of microbatches to constant 8
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:52 megatron_strategy:1045] Doing selective restore from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:52 pl:411] Loaded sharded_state_dict_metadata from checkpoint: {'distrib_optim_sharding_type': 'fully_sharded_model_space'}
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:52 pl:275] Using <megatron.core.dist_checkpointing.strategies.fully_parallel.FullyParallelLoadStrategyWrapper object at 0x7f20a18c62f0> dist-ckpt load strategy.
s_act_ckpt/0 [default0]:LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 14:56:52 serialization:184] DEPRECATED: Passing 'checkpoint_dir' as a Path object in load_common_state_dict will no longer be supported in a future release. Please pass it as a string instead.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 pl:297] Global Checkpoint Load : Rank : 0 : Start time : 1764079012.611s : Time spent in load_checkpoint: 0.744s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 megatron_strategy:1050] Restoring model weights from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 megatron_strategy:1058] Finished restoring from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True), cleaning up.
s_act_ckpt/0 [default0]:
s_act_ckpt/0 [default0]:  | Name   | Type     | Params | Mode 
s_act_ckpt/0 [default0]:--------------------------------------------
s_act_ckpt/0 [default0]:0 | module | GPTModel | 596 M  | train
s_act_ckpt/0 [default0]:--------------------------------------------
s_act_ckpt/0 [default0]:596 M     Trainable params
s_act_ckpt/0 [default0]:0         Non-trainable params
s_act_ckpt/0 [default0]:596 M     Total params
s_act_ckpt/0 [default0]:2,384.200 Total estimated model params size (MB)
s_act_ckpt/0 [default0]:569       Modules in train mode
s_act_ckpt/0 [default0]:0         Modules in eval mode
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 14:56:53 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=9` in the `DataLoader` to improve performance.
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.0.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.0.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.0.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.0.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.1.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.1.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.1.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.1.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.2.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.2.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.2.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.2.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.3.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.3.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.3.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.3.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.4.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.4.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.4.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.4.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.5.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.5.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.5.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.5.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.6.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.6.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.6.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.6.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.7.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.7.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.7.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.7.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.8.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.8.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.8.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.8.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.9.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.9.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.9.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.9.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.10.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.10.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.10.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.10.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.11.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.11.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.11.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.11.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.12.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.12.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.12.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.12.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.13.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.13.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.13.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.13.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.14.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.14.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.14.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.14.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.15.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.15.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.15.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.15.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.16.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.16.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.16.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.16.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.17.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.17.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.17.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.17.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.18.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.18.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.18.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.18.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.19.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.19.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.19.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.19.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.20.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.20.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.20.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.20.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.21.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.21.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.21.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.21.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.22.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.22.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.22.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.22.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.23.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.23.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.23.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.23.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.24.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.24.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.24.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.24.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.25.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.25.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.25.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.25.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.26.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.26.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.26.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.26.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.27.self_attention.linear_proj
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.27.self_attention.linear_qkv
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.27.mlp.linear_fc1
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 lora:468] Adding lora to: module.decoder.layers.27.mlp.linear_fc2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 model_transform:97] After applying model_transform:
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:  | Name   | Type     | Params | Mode 
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:--------------------------------------------
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:0 | module | GPTModel | 613 M  | train
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:--------------------------------------------
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:17.4 M    Trainable params
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:596 M     Non-trainable params
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:613 M     Total params
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:2,453.930 Total estimated model params size (MB)
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:1129      Modules in train mode
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:0         Modules in eval mode
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 peft:250] Initializing model parallel
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 megatron_parallel:584]  > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 613482496
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 megatron_parallel:587]  > number of trainable parameters: 17432576 (2.84% of total)
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 utils:673] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_megatron_fsdp=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache=False, nccl_ub=False, fsdp_double_buffer=False, outer_dp_sharding_strategy='no_shard', disable_symmetric_registration=False, delay_wgrad_compute=False)
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 utils:694] Number of buckets for gradient all-reduce / reduce-scatter: 1
s_act_ckpt/0 [default0]:    Params for bucket 1 (17432576 elements, 17432576 padded size):
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.adapter.linear_out.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.adapter.linear_in.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.adapter.linear_out.weight
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 peft:254] Setting up optimizers
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:56:53 utils:673] Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=None, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='delayed', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.98, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather=False, overlap_param_gather_with_optimizer_step=False, use_megatron_fsdp=False, optimizer_cpu_offload=False, optimizer_offload_fraction=0.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=False, timers=None, config_logger_dir='')
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 14:57:14 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('global_batch_size', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 14:57:14 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('val_loss', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 14:57:15 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=9` in the `DataLoader` to improve performance.
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 14:57:17 rerun_state_machine:1300] Implicit initialization of Rerun State Machine!
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 14:57:17 rerun_state_machine:238] RerunStateMachine initialized in mode disabled
s_act_ckpt/0 [default0]:Training epoch 0, iteration 0/999 | lr: 1.961e-06 | global_batch_size: 32 | global_step: 0 | reduced_train_loss: 1.102 | train_step_timing in s: 2.6
s_act_ckpt/0 [default0]:Training epoch 0, iteration 1/999 | lr: 3.922e-06 | global_batch_size: 32 | global_step: 1 | reduced_train_loss: 1.089 | train_step_timing in s: 2.71 | consumed_samples: 64
s_act_ckpt/0 [default0]:Training epoch 0, iteration 2/999 | lr: 5.882e-06 | global_batch_size: 32 | global_step: 2 | reduced_train_loss: 1.032 | train_step_timing in s: 2.463 | consumed_samples: 96
s_act_ckpt/0 [default0]:Training epoch 0, iteration 3/999 | lr: 7.843e-06 | global_batch_size: 32 | global_step: 3 | reduced_train_loss: 1.285 | train_step_timing in s: 2.33 | consumed_samples: 128
s_act_ckpt/0 [default0]:Training epoch 0, iteration 4/999 | lr: 9.804e-06 | global_batch_size: 32 | global_step: 4 | reduced_train_loss: 1.22 | train_step_timing in s: 2.546 | consumed_samples: 160
s_act_ckpt/0 [default0]:Training epoch 0, iteration 5/999 | lr: 1.176e-05 | global_batch_size: 32 | global_step: 5 | reduced_train_loss: 0.9833 | train_step_timing in s: 2.42 | consumed_samples: 192
s_act_ckpt/0 [default0]:Training epoch 0, iteration 6/999 | lr: 1.373e-05 | global_batch_size: 32 | global_step: 6 | reduced_train_loss: 0.9933 | train_step_timing in s: 2.476 | consumed_samples: 224
s_act_ckpt/0 [default0]:Training epoch 0, iteration 7/999 | lr: 1.569e-05 | global_batch_size: 32 | global_step: 7 | reduced_train_loss: 1.218 | train_step_timing in s: 2.428 | consumed_samples: 256
s_act_ckpt/0 [default0]:Training epoch 0, iteration 8/999 | lr: 1.765e-05 | global_batch_size: 32 | global_step: 8 | reduced_train_loss: 1.15 | train_step_timing in s: 2.352 | consumed_samples: 288
s_act_ckpt/0 [default0]:Training epoch 0, iteration 9/999 | lr: 1.961e-05 | global_batch_size: 32 | global_step: 9 | reduced_train_loss: 1.125 | train_step_timing in s: 2.461 | consumed_samples: 320
s_act_ckpt/0 [default0]:Training epoch 0, iteration 10/999 | lr: 2.157e-05 | global_batch_size: 32 | global_step: 10 | reduced_train_loss: 1.168 | train_step_timing in s: 2.325 | consumed_samples: 352
s_act_ckpt/0 [default0]:Training epoch 0, iteration 11/999 | lr: 2.353e-05 | global_batch_size: 32 | global_step: 11 | reduced_train_loss: 0.9788 | train_step_timing in s: 2.545 | consumed_samples: 384
s_act_ckpt/0 [default0]:Training epoch 0, iteration 12/999 | lr: 2.549e-05 | global_batch_size: 32 | global_step: 12 | reduced_train_loss: 0.9583 | train_step_timing in s: 2.508 | consumed_samples: 416
s_act_ckpt/0 [default0]:Training epoch 0, iteration 13/999 | lr: 2.745e-05 | global_batch_size: 32 | global_step: 13 | reduced_train_loss: 1.133 | train_step_timing in s: 2.447 | consumed_samples: 448
s_act_ckpt/0 [default0]:Training epoch 0, iteration 14/999 | lr: 2.941e-05 | global_batch_size: 32 | global_step: 14 | reduced_train_loss: 1.126 | train_step_timing in s: 2.511 | consumed_samples: 480
s_act_ckpt/0 [default0]:Training epoch 0, iteration 15/999 | lr: 3.137e-05 | global_batch_size: 32 | global_step: 15 | reduced_train_loss: 1.057 | train_step_timing in s: 2.573 | consumed_samples: 512
s_act_ckpt/0 [default0]:Training epoch 0, iteration 16/999 | lr: 3.333e-05 | global_batch_size: 32 | global_step: 16 | reduced_train_loss: 1.139 | train_step_timing in s: 2.401 | consumed_samples: 544
s_act_ckpt/0 [default0]:Training epoch 0, iteration 17/999 | lr: 3.529e-05 | global_batch_size: 32 | global_step: 17 | reduced_train_loss: 1.026 | train_step_timing in s: 2.433 | consumed_samples: 576
s_act_ckpt/0 [default0]:Training epoch 0, iteration 18/999 | lr: 3.725e-05 | global_batch_size: 32 | global_step: 18 | reduced_train_loss: 0.9653 | train_step_timing in s: 2.288 | consumed_samples: 608
s_act_ckpt/0 [default0]:Training epoch 0, iteration 19/999 | lr: 3.922e-05 | global_batch_size: 32 | global_step: 19 | reduced_train_loss: 1.024 | train_step_timing in s: 2.31 | consumed_samples: 640
s_act_ckpt/0 [default0]:Training epoch 0, iteration 20/999 | lr: 4.118e-05 | global_batch_size: 32 | global_step: 20 | reduced_train_loss: 1.371 | train_step_timing in s: 2.59 | consumed_samples: 672
s_act_ckpt/0 [default0]:Training epoch 0, iteration 21/999 | lr: 4.314e-05 | global_batch_size: 32 | global_step: 21 | reduced_train_loss: 1.079 | train_step_timing in s: 2.641 | consumed_samples: 704
s_act_ckpt/0 [default0]:Training epoch 0, iteration 22/999 | lr: 4.51e-05 | global_batch_size: 32 | global_step: 22 | reduced_train_loss: 1.194 | train_step_timing in s: 2.64 | consumed_samples: 736
s_act_ckpt/0 [default0]:Training epoch 0, iteration 23/999 | lr: 4.706e-05 | global_batch_size: 32 | global_step: 23 | reduced_train_loss: 1.121 | train_step_timing in s: 2.573 | consumed_samples: 768[default0]:
s_act_ckpt/0 [default0]:Training epoch 0, iteration 24/999 | lr: 4.902e-05 | global_batch_size: 32 | global_step: 24 | reduced_train_loss: 1.219 | train_step_timing in s: 2.435 | consumed_samples: 800
s_act_ckpt/0 [default0]:Training epoch 0, iteration 25/999 | lr: 5.098e-05 | global_batch_size: 32 | global_step: 25 | reduced_train_loss: 1.048 | train_step_timing in s: 2.498 | consumed_samples: 832
s_act_ckpt/0 [default0]:Training epoch 0, iteration 26/999 | lr: 5.294e-05 | global_batch_size: 32 | global_step: 26 | reduced_train_loss: 1.147 | train_step_timing in s: 2.518 | consumed_samples: 864
s_act_ckpt/0 [default0]:Training epoch 0, iteration 27/999 | lr: 5.49e-05 | global_batch_size: 32 | global_step: 27 | reduced_train_loss: 0.9749 | train_step_timing in s: 2.453 | consumed_samples: 896
s_act_ckpt/0 [default0]:Training epoch 0, iteration 28/999 | lr: 5.686e-05 | global_batch_size: 32 | global_step: 28 | reduced_train_loss: 1.122 | train_step_timing in s: 2.521 | consumed_samples: 928
s_act_ckpt/0 [default0]:Training epoch 0, iteration 29/999 | lr: 5.882e-05 | global_batch_size: 32 | global_step: 29 | reduced_train_loss: 1.068 | train_step_timing in s: 2.468 | consumed_samples: 960
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:59:28 pl:353] Using FullyParallelSaveStrategyWrapper(torch_dist, 1) dist-ckpt save strategy.
s_act_ckpt/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
s_act_ckpt/0 [default0]:  import pynvml  # type: ignore[import]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:59:34 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 30 : Start time: 1764079168.501s : Save duration: 5.871s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:59:34 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=29-consumed_samples=960.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 30/999 | lr: 6.078e-05 | global_batch_size: 32 | global_step: 30 | reduced_train_loss: 0.9728 | train_step_timing in s: 2.93 | consumed_samples: 992 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:59:37 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 31/999 | lr: 6.275e-05 | global_batch_size: 32 | global_step: 31 | reduced_train_loss: 1.234 | train_step_timing in s: 2.834 | consumed_samples: 1024 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:59:40 pl:222] Successfully saved checkpoint from iteration      30 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=29-consumed_samples=960.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:59:40 model_checkpoint:657] Async checkpoint save for step 30 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=29-consumed_samples=960.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 14:59:40 dist_ckpt_io:160] Async finalization time took 0.045 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 32/999 | lr: 6.471e-05 | global_batch_size: 32 | global_step: 32 | reduced_train_loss: 1.312 | train_step_timing in s: 2.585 | consumed_samples: 1056 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 33/999 | lr: 6.667e-05 | global_batch_size: 32 | global_step: 33 | reduced_train_loss: 0.9437 | train_step_timing in s: 2.671 | consumed_samples: 1088 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 34/999 | lr: 6.863e-05 | global_batch_size: 32 | global_step: 34 | reduced_train_loss: 1.18 | train_step_timing in s: 2.57 | consumed_samples: 1120 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 35/999 | lr: 7.059e-05 | global_batch_size: 32 | global_step: 35 | reduced_train_loss: 0.9088 | train_step_timing in s: 2.651 | consumed_samples: 1152 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 36/999 | lr: 7.255e-05 | global_batch_size: 32 | global_step: 36 | reduced_train_loss: 0.946 | train_step_timing in s: 2.597 | consumed_samples: 1184 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 37/999 | lr: 7.451e-05 | global_batch_size: 32 | global_step: 37 | reduced_train_loss: 1.156 | train_step_timing in s: 2.537 | consumed_samples: 1216 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 38/999 | lr: 7.647e-05 | global_batch_size: 32 | global_step: 38 | reduced_train_loss: 1.247 | train_step_timing in s: 2.398 | consumed_samples: 1248 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 39/999 | lr: 7.843e-05 | global_batch_size: 32 | global_step: 39 | reduced_train_loss: 1.017 | train_step_timing in s: 2.363 | consumed_samples: 1280 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 40/999 | lr: 8.039e-05 | global_batch_size: 32 | global_step: 40 | reduced_train_loss: 1.133 | train_step_timing in s: 2.388 | consumed_samples: 1312 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 41/999 | lr: 8.235e-05 | global_batch_size: 32 | global_step: 41 | reduced_train_loss: 1.2 | train_step_timing in s: 2.766 | consumed_samples: 1344 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 42/999 | lr: 8.431e-05 | global_batch_size: 32 | global_step: 42 | reduced_train_loss: 1.123 | train_step_timing in s: 2.436 | consumed_samples: 1376 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 43/999 | lr: 8.627e-05 | global_batch_size: 32 | global_step: 43 | reduced_train_loss: 1.123 | train_step_timing in s: 2.766 | consumed_samples: 1408 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 44/999 | lr: 8.824e-05 | global_batch_size: 32 | global_step: 44 | reduced_train_loss: 1.167 | train_step_timing in s: 2.764 | consumed_samples: 1440 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 45/999 | lr: 9.02e-05 | global_batch_size: 32 | global_step: 45 | reduced_train_loss: 1.062 | train_step_timing in s: 2.606 | consumed_samples: 1472 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 46/999 | lr: 9.216e-05 | global_batch_size: 32 | global_step: 46 | reduced_train_loss: 0.9559 | train_step_timing in s: 2.59 | consumed_samples: 1504 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 47/999 | lr: 9.412e-05 | global_batch_size: 32 | global_step: 47 | reduced_train_loss: 1.069 | train_step_timing in s: 2.568 | consumed_samples: 1536 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 48/999 | lr: 9.608e-05 | global_batch_size: 32 | global_step: 48 | reduced_train_loss: 1.043 | train_step_timing in s: 2.63 | consumed_samples: 1568 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 49/999 | lr: 9.804e-05 | global_batch_size: 32 | global_step: 49 | reduced_train_loss: 1.166 | train_step_timing in s: 2.509 | consumed_samples: 1600 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 49: 'val_loss' reached 1.16458 (best 1.16458), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=49-consumed_samples=1600.0.ckpt' as top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:00:27 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 49 : Start time: 1764079227.384s : Save duration: 0.165s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:00:27 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=49-consumed_samples=1600.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:00:27 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 50/999 | lr: 0.0001 | global_batch_size: 32 | global_step: 50 | reduced_train_loss: 1.134 | train_step_timing in s: 2.739 | consumed_samples: 1632 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:00:30 pl:222] Successfully saved checkpoint from iteration      49 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=49-consumed_samples=1600.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:00:30 model_checkpoint:657] Async checkpoint save for step 50 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=49-consumed_samples=1600.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:00:30 dist_ckpt_io:160] Async finalization time took 0.112 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 51/999 | lr: 0.0001 | global_batch_size: 32 | global_step: 51 | reduced_train_loss: 1.192 | train_step_timing in s: 2.292 | consumed_samples: 1664 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 52/999 | lr: 0.0001 | global_batch_size: 32 | global_step: 52 | reduced_train_loss: 0.9092 | train_step_timing in s: 2.284 | consumed_samples: 1696 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 53/999 | lr: 0.0001 | global_batch_size: 32 | global_step: 53 | reduced_train_loss: 1.179 | train_step_timing in s: 2.562 | consumed_samples: 1728 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 54/999 | lr: 0.0001 | global_batch_size: 32 | global_step: 54 | reduced_train_loss: 1.123 | train_step_timing in s: 2.262 | consumed_samples: 1760 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 55/999 | lr: 9.999e-05 | global_batch_size: 32 | global_step: 55 | reduced_train_loss: 1.257 | train_step_timing in s: 2.634 | consumed_samples: 1792 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 56/999 | lr: 9.999e-05 | global_batch_size: 32 | global_step: 56 | reduced_train_loss: 1.184 | train_step_timing in s: 2.618 | consumed_samples: 1824 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 57/999 | lr: 9.999e-05 | global_batch_size: 32 | global_step: 57 | reduced_train_loss: 1.099 | train_step_timing in s: 2.511 | consumed_samples: 1856 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 58/999 | lr: 9.998e-05 | global_batch_size: 32 | global_step: 58 | reduced_train_loss: 1.095 | train_step_timing in s: 2.438 | consumed_samples: 1888 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 59/999 | lr: 9.998e-05 | global_batch_size: 32 | global_step: 59 | reduced_train_loss: 1.159 | train_step_timing in s: 2.709 | consumed_samples: 1920 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:01:52 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 60 : Start time: 1764079311.963s : Save duration: 0.165s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:01:52 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=59-consumed_samples=1920.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 60/999 | lr: 9.997e-05 | global_batch_size: 32 | global_step: 60 | reduced_train_loss: 1.374 | train_step_timing in s: 2.643 | consumed_samples: 1952 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:01:55 pl:222] Successfully saved checkpoint from iteration      60 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=59-consumed_samples=1920.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:01:55 model_checkpoint:657] Async checkpoint save for step 60 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=59-consumed_samples=1920.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:01:55 dist_ckpt_io:160] Async finalization time took 0.052 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 61/999 | lr: 9.997e-05 | global_batch_size: 32 | global_step: 61 | reduced_train_loss: 1.138 | train_step_timing in s: 2.612 | consumed_samples: 1984 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 62/999 | lr: 9.996e-05 | global_batch_size: 32 | global_step: 62 | reduced_train_loss: 1.137 | train_step_timing in s: 2.348 | consumed_samples: 2016 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 63/999 | lr: 9.995e-05 | global_batch_size: 32 | global_step: 63 | reduced_train_loss: 0.9994 | train_step_timing in s: 2.641 | consumed_samples: 2048 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 64/999 | lr: 9.995e-05 | global_batch_size: 32 | global_step: 64 | reduced_train_loss: 1.17 | train_step_timing in s: 2.563 | consumed_samples: 2080 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 65/999 | lr: 9.994e-05 | global_batch_size: 32 | global_step: 65 | reduced_train_loss: 1.034 | train_step_timing in s: 2.902 | consumed_samples: 2112 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 66/999 | lr: 9.993e-05 | global_batch_size: 32 | global_step: 66 | reduced_train_loss: 1.206 | train_step_timing in s: 2.77 | consumed_samples: 2144 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 67/999 | lr: 9.992e-05 | global_batch_size: 32 | global_step: 67 | reduced_train_loss: 1.159 | train_step_timing in s: 2.716 | consumed_samples: 2176 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 68/999 | lr: 9.991e-05 | global_batch_size: 32 | global_step: 68 | reduced_train_loss: 1.107 | train_step_timing in s: 2.538 | consumed_samples: 2208 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 69/999 | lr: 9.99e-05 | global_batch_size: 32 | global_step: 69 | reduced_train_loss: 1.135 | train_step_timing in s: 2.277 | consumed_samples: 2240 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 70/999 | lr: 9.989e-05 | global_batch_size: 32 | global_step: 70 | reduced_train_loss: 1.219 | train_step_timing in s: 2.292 | consumed_samples: 2272 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 71/999 | lr: 9.988e-05 | global_batch_size: 32 | global_step: 71 | reduced_train_loss: 1.056 | train_step_timing in s: 2.264 | consumed_samples: 2304 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 72/999 | lr: 9.987e-05 | global_batch_size: 32 | global_step: 72 | reduced_train_loss: 1.205 | train_step_timing in s: 2.726 | consumed_samples: 2336 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 73/999 | lr: 9.986e-05 | global_batch_size: 32 | global_step: 73 | reduced_train_loss: 1.135 | train_step_timing in s: 2.592 | consumed_samples: 2368 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 74/999 | lr: 9.984e-05 | global_batch_size: 32 | global_step: 74 | reduced_train_loss: 1.068 | train_step_timing in s: 2.338 | consumed_samples: 2400 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 75/999 | lr: 9.983e-05 | global_batch_size: 32 | global_step: 75 | reduced_train_loss: 1.381 | train_step_timing in s: 2.683 | consumed_samples: 2432 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 76/999 | lr: 9.982e-05 | global_batch_size: 32 | global_step: 76 | reduced_train_loss: 1.114 | train_step_timing in s: 2.45 | consumed_samples: 2464 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 77/999 | lr: 9.98e-05 | global_batch_size: 32 | global_step: 77 | reduced_train_loss: 0.934 | train_step_timing in s: 2.713 | consumed_samples: 2496 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 78/999 | lr: 9.979e-05 | global_batch_size: 32 | global_step: 78 | reduced_train_loss: 1.07 | train_step_timing in s: 2.613 | consumed_samples: 2528 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 79/999 | lr: 9.977e-05 | global_batch_size: 32 | global_step: 79 | reduced_train_loss: 1.229 | train_step_timing in s: 2.315 | consumed_samples: 2560 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 80/999 | lr: 9.975e-05 | global_batch_size: 32 | global_step: 80 | reduced_train_loss: 0.9949 | train_step_timing in s: 2.64 | consumed_samples: 2592 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 81/999 | lr: 9.974e-05 | global_batch_size: 32 | global_step: 81 | reduced_train_loss: 1.148 | train_step_timing in s: 2.602 | consumed_samples: 2624 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 82/999 | lr: 9.972e-05 | global_batch_size: 32 | global_step: 82 | reduced_train_loss: 1.001 | train_step_timing in s: 2.673 | consumed_samples: 2656 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 83/999 | lr: 9.97e-05 | global_batch_size: 32 | global_step: 83 | reduced_train_loss: 1.238 | train_step_timing in s: 2.55 | consumed_samples: 2688 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 84/999 | lr: 9.968e-05 | global_batch_size: 32 | global_step: 84 | reduced_train_loss: 1.203 | train_step_timing in s: 2.307 | consumed_samples: 2720 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 85/999 | lr: 9.967e-05 | global_batch_size: 32 | global_step: 85 | reduced_train_loss: 1.26 | train_step_timing in s: 2.231 | consumed_samples: 2752 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 86/999 | lr: 9.965e-05 | global_batch_size: 32 | global_step: 86 | reduced_train_loss: 1.089 | train_step_timing in s: 2.298 | consumed_samples: 2784 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 87/999 | lr: 9.963e-05 | global_batch_size: 32 | global_step: 87 | reduced_train_loss: 1.072 | train_step_timing in s: 2.314 | consumed_samples: 2816 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 88/999 | lr: 9.961e-05 | global_batch_size: 32 | global_step: 88 | reduced_train_loss: 1.174 | train_step_timing in s: 2.621 | consumed_samples: 2848 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 89/999 | lr: 9.958e-05 | global_batch_size: 32 | global_step: 89 | reduced_train_loss: 0.9827 | train_step_timing in s: 2.4 | consumed_samples: 2880 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:10 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 90 : Start time: 1764079449.834s : Save duration: 0.168s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:10 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=89-consumed_samples=2880.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 90/999 | lr: 9.956e-05 | global_batch_size: 32 | global_step: 90 | reduced_train_loss: 1.083 | train_step_timing in s: 2.692 | consumed_samples: 2912 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:13 pl:222] Successfully saved checkpoint from iteration      90 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=89-consumed_samples=2880.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:13 model_checkpoint:657] Async checkpoint save for step 90 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=89-consumed_samples=2880.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:13 dist_ckpt_io:160] Async finalization time took 0.108 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 91/999 | lr: 9.954e-05 | global_batch_size: 32 | global_step: 91 | reduced_train_loss: 1.249 | train_step_timing in s: 2.3 | consumed_samples: 2944 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 92/999 | lr: 9.952e-05 | global_batch_size: 32 | global_step: 92 | reduced_train_loss: 1.165 | train_step_timing in s: 2.325 | consumed_samples: 2976 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 93/999 | lr: 9.95e-05 | global_batch_size: 32 | global_step: 93 | reduced_train_loss: 1.012 | train_step_timing in s: 2.291 | consumed_samples: 3008 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 94/999 | lr: 9.947e-05 | global_batch_size: 32 | global_step: 94 | reduced_train_loss: 1.051 | train_step_timing in s: 2.62 | consumed_samples: 3040 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 95/999 | lr: 9.945e-05 | global_batch_size: 32 | global_step: 95 | reduced_train_loss: 1.154 | train_step_timing in s: 2.331 | consumed_samples: 3072 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 96/999 | lr: 9.942e-05 | global_batch_size: 32 | global_step: 96 | reduced_train_loss: 1.11 | train_step_timing in s: 2.635 | consumed_samples: 3104 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 97/999 | lr: 9.94e-05 | global_batch_size: 32 | global_step: 97 | reduced_train_loss: 1.185 | train_step_timing in s: 2.642 | consumed_samples: 3136 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 98/999 | lr: 9.937e-05 | global_batch_size: 32 | global_step: 98 | reduced_train_loss: 1.072 | train_step_timing in s: 2.655 | consumed_samples: 3168 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 99/999 | lr: 9.935e-05 | global_batch_size: 32 | global_step: 99 | reduced_train_loss: 1.046 | train_step_timing in s: 2.636 | consumed_samples: 3200 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 99: 'val_loss' reached 1.16458 (best 1.16458), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=99-consumed_samples=3200.0.ckpt' as top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:36 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 99 : Start time: 1764079475.795s : Save duration: 0.644s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:36 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=99-consumed_samples=3200.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:36 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 100/999 | lr: 9.932e-05 | global_batch_size: 32 | global_step: 100 | reduced_train_loss: 1.018 | train_step_timing in s: 2.956 | consumed_samples: 3232 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:39 pl:222] Successfully saved checkpoint from iteration      99 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=99-consumed_samples=3200.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:39 model_checkpoint:657] Async checkpoint save for step 100 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=99-consumed_samples=3200.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:04:39 dist_ckpt_io:160] Async finalization time took 0.111 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 101/999 | lr: 9.929e-05 | global_batch_size: 32 | global_step: 101 | reduced_train_loss: 0.9265 | train_step_timing in s: 2.393 | consumed_samples: 3264 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 102/999 | lr: 9.926e-05 | global_batch_size: 32 | global_step: 102 | reduced_train_loss: 1.355 | train_step_timing in s: 2.335 | consumed_samples: 3296 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 103/999 | lr: 9.923e-05 | global_batch_size: 32 | global_step: 103 | reduced_train_loss: 0.9501 | train_step_timing in s: 2.386 | consumed_samples: 3328 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 104/999 | lr: 9.92e-05 | global_batch_size: 32 | global_step: 104 | reduced_train_loss: 1.257 | train_step_timing in s: 2.365 | consumed_samples: 3360 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 105/999 | lr: 9.918e-05 | global_batch_size: 32 | global_step: 105 | reduced_train_loss: 1.126 | train_step_timing in s: 2.256 | consumed_samples: 3392 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 106/999 | lr: 9.915e-05 | global_batch_size: 32 | global_step: 106 | reduced_train_loss: 1.171 | train_step_timing in s: 2.249 | consumed_samples: 3424 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 107/999 | lr: 9.911e-05 | global_batch_size: 32 | global_step: 107 | reduced_train_loss: 1.118 | train_step_timing in s: 2.318 | consumed_samples: 3456 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 108/999 | lr: 9.908e-05 | global_batch_size: 32 | global_step: 108 | reduced_train_loss: 1.081 | train_step_timing in s: 2.258 | consumed_samples: 3488 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 109/999 | lr: 9.905e-05 | global_batch_size: 32 | global_step: 109 | reduced_train_loss: 1.227 | train_step_timing in s: 2.365 | consumed_samples: 3520 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 110/999 | lr: 9.902e-05 | global_batch_size: 32 | global_step: 110 | reduced_train_loss: 0.9809 | train_step_timing in s: 2.314 | consumed_samples: 3552 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 111/999 | lr: 9.899e-05 | global_batch_size: 32 | global_step: 111 | reduced_train_loss: 1.192 | train_step_timing in s: 2.339 | consumed_samples: 3584 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 112/999 | lr: 9.895e-05 | global_batch_size: 32 | global_step: 112 | reduced_train_loss: 1.079 | train_step_timing in s: 2.37 | consumed_samples: 3616 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 113/999 | lr: 9.892e-05 | global_batch_size: 32 | global_step: 113 | reduced_train_loss: 1.162 | train_step_timing in s: 2.406 | consumed_samples: 3648 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 114/999 | lr: 9.888e-05 | global_batch_size: 32 | global_step: 114 | reduced_train_loss: 1.218 | train_step_timing in s: 2.352 | consumed_samples: 3680 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 115/999 | lr: 9.885e-05 | global_batch_size: 32 | global_step: 115 | reduced_train_loss: 0.9075 | train_step_timing in s: 2.334 | consumed_samples: 3712 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 116/999 | lr: 9.881e-05 | global_batch_size: 32 | global_step: 116 | reduced_train_loss: 1.14 | train_step_timing in s: 2.401 | consumed_samples: 3744 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 117/999 | lr: 9.878e-05 | global_batch_size: 32 | global_step: 117 | reduced_train_loss: 1.312 | train_step_timing in s: 2.328 | consumed_samples: 3776 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 118/999 | lr: 9.874e-05 | global_batch_size: 32 | global_step: 118 | reduced_train_loss: 1.127 | train_step_timing in s: 2.389 | consumed_samples: 3808 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 119/999 | lr: 9.87e-05 | global_batch_size: 32 | global_step: 119 | reduced_train_loss: 1.102 | train_step_timing in s: 2.36 | consumed_samples: 3840 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:06:20 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 120 : Start time: 1764079579.979s : Save duration: 0.168s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:06:20 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=119-consumed_samples=3840.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 120/999 | lr: 9.867e-05 | global_batch_size: 32 | global_step: 120 | reduced_train_loss: 1.18 | train_step_timing in s: 2.847 | consumed_samples: 3872 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:06:23 pl:222] Successfully saved checkpoint from iteration     120 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=119-consumed_samples=3840.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:06:23 model_checkpoint:657] Async checkpoint save for step 120 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=119-consumed_samples=3840.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:06:23 dist_ckpt_io:160] Async finalization time took 0.056 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 121/999 | lr: 9.863e-05 | global_batch_size: 32 | global_step: 121 | reduced_train_loss: 1.157 | train_step_timing in s: 2.382 | consumed_samples: 3904 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 122/999 | lr: 9.859e-05 | global_batch_size: 32 | global_step: 122 | reduced_train_loss: 1.068 | train_step_timing in s: 2.346 | consumed_samples: 3936 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 123/999 | lr: 9.855e-05 | global_batch_size: 32 | global_step: 123 | reduced_train_loss: 1.151 | train_step_timing in s: 2.323 | consumed_samples: 3968 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 124/999 | lr: 9.851e-05 | global_batch_size: 32 | global_step: 124 | reduced_train_loss: 1.089 | train_step_timing in s: 2.402 | consumed_samples: 4000 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 125/999 | lr: 9.847e-05 | global_batch_size: 32 | global_step: 125 | reduced_train_loss: 1.131 | train_step_timing in s: 2.345 | consumed_samples: 4032 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 126/999 | lr: 9.843e-05 | global_batch_size: 32 | global_step: 126 | reduced_train_loss: 1.192 | train_step_timing in s: 2.358 | consumed_samples: 4064 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 127/999 | lr: 9.839e-05 | global_batch_size: 32 | global_step: 127 | reduced_train_loss: 1.043 | train_step_timing in s: 2.373 | consumed_samples: 4096 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 128/999 | lr: 9.835e-05 | global_batch_size: 32 | global_step: 128 | reduced_train_loss: 1.164 | train_step_timing in s: 2.338 | consumed_samples: 4128 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 129/999 | lr: 9.83e-05 | global_batch_size: 32 | global_step: 129 | reduced_train_loss: 1.238 | train_step_timing in s: 2.237 | consumed_samples: 4160 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 130/999 | lr: 9.826e-05 | global_batch_size: 32 | global_step: 130 | reduced_train_loss: 1.138 | train_step_timing in s: 2.274 | consumed_samples: 4192 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 131/999 | lr: 9.822e-05 | global_batch_size: 32 | global_step: 131 | reduced_train_loss: 1.071 | train_step_timing in s: 2.225 | consumed_samples: 4224 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 132/999 | lr: 9.817e-05 | global_batch_size: 32 | global_step: 132 | reduced_train_loss: 1.12 | train_step_timing in s: 2.254 | consumed_samples: 4256 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 133/999 | lr: 9.813e-05 | global_batch_size: 32 | global_step: 133 | reduced_train_loss: 1.184 | train_step_timing in s: 2.405 | consumed_samples: 4288 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 134/999 | lr: 9.808e-05 | global_batch_size: 32 | global_step: 134 | reduced_train_loss: 1.218 | train_step_timing in s: 2.31 | consumed_samples: 4320 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 135/999 | lr: 9.804e-05 | global_batch_size: 32 | global_step: 135 | reduced_train_loss: 1.089 | train_step_timing in s: 2.394 | consumed_samples: 4352 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 136/999 | lr: 9.799e-05 | global_batch_size: 32 | global_step: 136 | reduced_train_loss: 1.089 | train_step_timing in s: 2.37 | consumed_samples: 4384 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 137/999 | lr: 9.794e-05 | global_batch_size: 32 | global_step: 137 | reduced_train_loss: 0.9014 | train_step_timing in s: 2.33 | consumed_samples: 4416 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 138/999 | lr: 9.79e-05 | global_batch_size: 32 | global_step: 138 | reduced_train_loss: 1.039 | train_step_timing in s: 2.4 | consumed_samples: 4448 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 139/999 | lr: 9.785e-05 | global_batch_size: 32 | global_step: 139 | reduced_train_loss: 1.158 | train_step_timing in s: 2.287 | consumed_samples: 4480 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 140/999 | lr: 9.78e-05 | global_batch_size: 32 | global_step: 140 | reduced_train_loss: 1.148 | train_step_timing in s: 2.369 | consumed_samples: 4512 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 141/999 | lr: 9.775e-05 | global_batch_size: 32 | global_step: 141 | reduced_train_loss: 1.361 | train_step_timing in s: 2.411 | consumed_samples: 4544 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 142/999 | lr: 9.77e-05 | global_batch_size: 32 | global_step: 142 | reduced_train_loss: 1.032 | train_step_timing in s: 2.371 | consumed_samples: 4576 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 143/999 | lr: 9.765e-05 | global_batch_size: 32 | global_step: 143 | reduced_train_loss: 1.077 | train_step_timing in s: 2.404 | consumed_samples: 4608 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 144/999 | lr: 9.76e-05 | global_batch_size: 32 | global_step: 144 | reduced_train_loss: 1.235 | train_step_timing in s: 2.354 | consumed_samples: 4640 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 145/999 | lr: 9.755e-05 | global_batch_size: 32 | global_step: 145 | reduced_train_loss: 1.117 | train_step_timing in s: 2.225 | consumed_samples: 4672 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 146/999 | lr: 9.75e-05 | global_batch_size: 32 | global_step: 146 | reduced_train_loss: 0.975 | train_step_timing in s: 2.251 | consumed_samples: 4704 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 147/999 | lr: 9.745e-05 | global_batch_size: 32 | global_step: 147 | reduced_train_loss: 1.239 | train_step_timing in s: 2.262 | consumed_samples: 4736 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 148/999 | lr: 9.74e-05 | global_batch_size: 32 | global_step: 148 | reduced_train_loss: 0.9576 | train_step_timing in s: 2.279 | consumed_samples: 4768 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 149/999 | lr: 9.734e-05 | global_batch_size: 32 | global_step: 149 | reduced_train_loss: 1.095 | train_step_timing in s: 2.407 | consumed_samples: 4800 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 149: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:07:31 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 149 : Start time: 1764079651.635s : Save duration: 0.177s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:07:32 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=149-consumed_samples=4800.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:07:32 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:Training epoch 0, iteration 150/999 | lr: 9.729e-05 | global_batch_size: 32 | global_step: 150 | reduced_train_loss: 1.039 | train_step_timing in s: 2.332 | consumed_samples: 4832 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:08:32 pl:222] Successfully saved checkpoint from iteration     149 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=149-consumed_samples=4800.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:08:32 model_checkpoint:657] Async checkpoint save for step 150 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=149-consumed_samples=4800.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:08:32 dist_ckpt_io:160] Async finalization time took 0.121 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 151/999 | lr: 9.724e-05 | global_batch_size: 32 | global_step: 151 | reduced_train_loss: 1.198 | train_step_timing in s: 2.23 | consumed_samples: 4864 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 152/999 | lr: 9.718e-05 | global_batch_size: 32 | global_step: 152 | reduced_train_loss: 1.011 | train_step_timing in s: 2.364 | consumed_samples: 4896 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 153/999 | lr: 9.713e-05 | global_batch_size: 32 | global_step: 153 | reduced_train_loss: 1.126 | train_step_timing in s: 2.337 | consumed_samples: 4928 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 154/999 | lr: 9.707e-05 | global_batch_size: 32 | global_step: 154 | reduced_train_loss: 1.286 | train_step_timing in s: 2.254 | consumed_samples: 4960 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 155/999 | lr: 9.702e-05 | global_batch_size: 32 | global_step: 155 | reduced_train_loss: 1.136 | train_step_timing in s: 2.395 | consumed_samples: 4992 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 156/999 | lr: 9.696e-05 | global_batch_size: 32 | global_step: 156 | reduced_train_loss: 1.108 | train_step_timing in s: 2.281 | consumed_samples: 5024 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 157/999 | lr: 9.69e-05 | global_batch_size: 32 | global_step: 157 | reduced_train_loss: 1.089 | train_step_timing in s: 2.363 | consumed_samples: 5056 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 158/999 | lr: 9.684e-05 | global_batch_size: 32 | global_step: 158 | reduced_train_loss: 1.169 | train_step_timing in s: 2.332 | consumed_samples: 5088 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 159/999 | lr: 9.679e-05 | global_batch_size: 32 | global_step: 159 | reduced_train_loss: 1.198 | train_step_timing in s: 2.249 | consumed_samples: 5120 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 160/999 | lr: 9.673e-05 | global_batch_size: 32 | global_step: 160 | reduced_train_loss: 1.146 | train_step_timing in s: 2.36 | consumed_samples: 5152 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 161/999 | lr: 9.667e-05 | global_batch_size: 32 | global_step: 161 | reduced_train_loss: 1.129 | train_step_timing in s: 2.351 | consumed_samples: 5184 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 162/999 | lr: 9.661e-05 | global_batch_size: 32 | global_step: 162 | reduced_train_loss: 1.004 | train_step_timing in s: 2.366 | consumed_samples: 5216 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 163/999 | lr: 9.655e-05 | global_batch_size: 32 | global_step: 163 | reduced_train_loss: 1.03 | train_step_timing in s: 2.386 | consumed_samples: 5248 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 164/999 | lr: 9.649e-05 | global_batch_size: 32 | global_step: 164 | reduced_train_loss: 1.059 | train_step_timing in s: 2.388 | consumed_samples: 5280 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 165/999 | lr: 9.643e-05 | global_batch_size: 32 | global_step: 165 | reduced_train_loss: 1.236 | train_step_timing in s: 2.196 | consumed_samples: 5312 | val_loss: 1.165[default0]:
s_act_ckpt/0 [default0]:Training epoch 0, iteration 166/999 | lr: 9.637e-05 | global_batch_size: 32 | global_step: 166 | reduced_train_loss: 1.165 | train_step_timing in s: 2.205 | consumed_samples: 5344 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 167/999 | lr: 9.63e-05 | global_batch_size: 32 | global_step: 167 | reduced_train_loss: 1.09 | train_step_timing in s: 2.304 | consumed_samples: 5376 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 168/999 | lr: 9.624e-05 | global_batch_size: 32 | global_step: 168 | reduced_train_loss: 1.12 | train_step_timing in s: 2.381 | consumed_samples: 5408 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 169/999 | lr: 9.618e-05 | global_batch_size: 32 | global_step: 169 | reduced_train_loss: 0.9905 | train_step_timing in s: 2.37 | consumed_samples: 5440 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 170/999 | lr: 9.611e-05 | global_batch_size: 32 | global_step: 170 | reduced_train_loss: 1.188 | train_step_timing in s: 2.29 | consumed_samples: 5472 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 171/999 | lr: 9.605e-05 | global_batch_size: 32 | global_step: 171 | reduced_train_loss: 1.053 | train_step_timing in s: 2.397 | consumed_samples: 5504 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 172/999 | lr: 9.599e-05 | global_batch_size: 32 | global_step: 172 | reduced_train_loss: 1.079 | train_step_timing in s: 2.28 | consumed_samples: 5536 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 173/999 | lr: 9.592e-05 | global_batch_size: 32 | global_step: 173 | reduced_train_loss: 1.207 | train_step_timing in s: 2.427 | consumed_samples: 5568 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 174/999 | lr: 9.585e-05 | global_batch_size: 32 | global_step: 174 | reduced_train_loss: 1.009 | train_step_timing in s: 2.365 | consumed_samples: 5600 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 175/999 | lr: 9.579e-05 | global_batch_size: 32 | global_step: 175 | reduced_train_loss: 1.153 | train_step_timing in s: 2.273 | consumed_samples: 5632 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 176/999 | lr: 9.572e-05 | global_batch_size: 32 | global_step: 176 | reduced_train_loss: 1.078 | train_step_timing in s: 2.342 | consumed_samples: 5664 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 177/999 | lr: 9.565e-05 | global_batch_size: 32 | global_step: 177 | reduced_train_loss: 1.2 | train_step_timing in s: 2.343 | consumed_samples: 5696 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 178/999 | lr: 9.559e-05 | global_batch_size: 32 | global_step: 178 | reduced_train_loss: 0.9602 | train_step_timing in s: 2.38 | consumed_samples: 5728 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 179/999 | lr: 9.552e-05 | global_batch_size: 32 | global_step: 179 | reduced_train_loss: 1.163 | train_step_timing in s: 2.325 | consumed_samples: 5760 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:10:36 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 180 : Start time: 1764079835.993s : Save duration: 0.750s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:10:37 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=179-consumed_samples=5760.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 180/999 | lr: 9.545e-05 | global_batch_size: 32 | global_step: 180 | reduced_train_loss: 1.14 | train_step_timing in s: 2.922 | consumed_samples: 5792 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:10:40 pl:222] Successfully saved checkpoint from iteration     180 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=179-consumed_samples=5760.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:10:40 model_checkpoint:657] Async checkpoint save for step 180 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=179-consumed_samples=5760.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:10:40 dist_ckpt_io:160] Async finalization time took 0.114 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 181/999 | lr: 9.538e-05 | global_batch_size: 32 | global_step: 181 | reduced_train_loss: 1.199 | train_step_timing in s: 2.449 | consumed_samples: 5824 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 182/999 | lr: 9.531e-05 | global_batch_size: 32 | global_step: 182 | reduced_train_loss: 1.179 | train_step_timing in s: 2.393 | consumed_samples: 5856 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 183/999 | lr: 9.524e-05 | global_batch_size: 32 | global_step: 183 | reduced_train_loss: 1.243 | train_step_timing in s: 2.432 | consumed_samples: 5888 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 184/999 | lr: 9.517e-05 | global_batch_size: 32 | global_step: 184 | reduced_train_loss: 0.8884 | train_step_timing in s: 2.339 | consumed_samples: 5920 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 185/999 | lr: 9.51e-05 | global_batch_size: 32 | global_step: 185 | reduced_train_loss: 1.184 | train_step_timing in s: 2.386 | consumed_samples: 5952 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 186/999 | lr: 9.503e-05 | global_batch_size: 32 | global_step: 186 | reduced_train_loss: 1.432 | train_step_timing in s: 2.319 | consumed_samples: 5984 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 187/999 | lr: 9.496e-05 | global_batch_size: 32 | global_step: 187 | reduced_train_loss: 1.043 | train_step_timing in s: 2.263 | consumed_samples: 6016 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 188/999 | lr: 9.488e-05 | global_batch_size: 32 | global_step: 188 | reduced_train_loss: 1.314 | train_step_timing in s: 2.262 | consumed_samples: 6048 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 189/999 | lr: 9.481e-05 | global_batch_size: 32 | global_step: 189 | reduced_train_loss: 1.333 | train_step_timing in s: 2.254 | consumed_samples: 6080 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 190/999 | lr: 9.474e-05 | global_batch_size: 32 | global_step: 190 | reduced_train_loss: 1.365 | train_step_timing in s: 2.325 | consumed_samples: 6112 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 191/999 | lr: 9.466e-05 | global_batch_size: 32 | global_step: 191 | reduced_train_loss: 1.199 | train_step_timing in s: 2.347 | consumed_samples: 6144 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 192/999 | lr: 9.459e-05 | global_batch_size: 32 | global_step: 192 | reduced_train_loss: 1.29 | train_step_timing in s: 2.403 | consumed_samples: 6176 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 193/999 | lr: 9.451e-05 | global_batch_size: 32 | global_step: 193 | reduced_train_loss: 1.19 | train_step_timing in s: 2.364 | consumed_samples: 6208 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 194/999 | lr: 9.444e-05 | global_batch_size: 32 | global_step: 194 | reduced_train_loss: 1.09 | train_step_timing in s: 2.365 | consumed_samples: 6240 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 195/999 | lr: 9.436e-05 | global_batch_size: 32 | global_step: 195 | reduced_train_loss: 1.157 | train_step_timing in s: 2.358 | consumed_samples: 6272 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 196/999 | lr: 9.428e-05 | global_batch_size: 32 | global_step: 196 | reduced_train_loss: 1.081 | train_step_timing in s: 2.408 | consumed_samples: 6304 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 197/999 | lr: 9.421e-05 | global_batch_size: 32 | global_step: 197 | reduced_train_loss: 1.197 | train_step_timing in s: 2.418 | consumed_samples: 6336 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 198/999 | lr: 9.413e-05 | global_batch_size: 32 | global_step: 198 | reduced_train_loss: 1.124 | train_step_timing in s: 2.43 | consumed_samples: 6368 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 199: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:Training epoch 0, iteration 199/999 | lr: 9.405e-05 | global_batch_size: 32 | global_step: 199 | reduced_train_loss: 1.091 | train_step_timing in s: 2.364 | consumed_samples: 6400 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:11:25 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 199 : Start time: 1764079885.377s : Save duration: 0.190s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:11:25 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=199-consumed_samples=6400.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:11:25 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 200/999 | lr: 9.397e-05 | global_batch_size: 32 | global_step: 200 | reduced_train_loss: 1.038 | train_step_timing in s: 2.662 | consumed_samples: 6432 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:11:28 pl:222] Successfully saved checkpoint from iteration     199 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=199-consumed_samples=6400.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:11:28 model_checkpoint:657] Async checkpoint save for step 200 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=199-consumed_samples=6400.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:11:28 dist_ckpt_io:160] Async finalization time took 0.109 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 201/999 | lr: 9.389e-05 | global_batch_size: 32 | global_step: 201 | reduced_train_loss: 0.9472 | train_step_timing in s: 2.216 | consumed_samples: 6464 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 202/999 | lr: 9.382e-05 | global_batch_size: 32 | global_step: 202 | reduced_train_loss: 1.218 | train_step_timing in s: 2.285 | consumed_samples: 6496 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 203/999 | lr: 9.374e-05 | global_batch_size: 32 | global_step: 203 | reduced_train_loss: 1.179 | train_step_timing in s: 2.283 | consumed_samples: 6528 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 204/999 | lr: 9.366e-05 | global_batch_size: 32 | global_step: 204 | reduced_train_loss: 1.343 | train_step_timing in s: 2.311 | consumed_samples: 6560 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 205/999 | lr: 9.357e-05 | global_batch_size: 32 | global_step: 205 | reduced_train_loss: 1.17 | train_step_timing in s: 2.286 | consumed_samples: 6592 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 206/999 | lr: 9.349e-05 | global_batch_size: 32 | global_step: 206 | reduced_train_loss: 1.07 | train_step_timing in s: 2.37 | consumed_samples: 6624 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 207/999 | lr: 9.341e-05 | global_batch_size: 32 | global_step: 207 | reduced_train_loss: 1.093 | train_step_timing in s: 2.34 | consumed_samples: 6656 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 208/999 | lr: 9.333e-05 | global_batch_size: 32 | global_step: 208 | reduced_train_loss: 1.041 | train_step_timing in s: 2.381 | consumed_samples: 6688 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 209/999 | lr: 9.325e-05 | global_batch_size: 32 | global_step: 209 | reduced_train_loss: 1.167 | train_step_timing in s: 2.368 | consumed_samples: 6720 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:12:45 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 210 : Start time: 1764079965.465s : Save duration: 0.168s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:12:45 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=209-consumed_samples=6720.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 210/999 | lr: 9.316e-05 | global_batch_size: 32 | global_step: 210 | reduced_train_loss: 1.242 | train_step_timing in s: 2.648 | consumed_samples: 6752 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:12:48 pl:222] Successfully saved checkpoint from iteration     210 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=209-consumed_samples=6720.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:12:48 model_checkpoint:657] Async checkpoint save for step 210 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=209-consumed_samples=6720.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:12:48 dist_ckpt_io:160] Async finalization time took 0.110 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 211/999 | lr: 9.308e-05 | global_batch_size: 32 | global_step: 211 | reduced_train_loss: 1.078 | train_step_timing in s: 2.386 | consumed_samples: 6784 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 212/999 | lr: 9.299e-05 | global_batch_size: 32 | global_step: 212 | reduced_train_loss: 1.198 | train_step_timing in s: 2.355 | consumed_samples: 6816 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 213/999 | lr: 9.291e-05 | global_batch_size: 32 | global_step: 213 | reduced_train_loss: 1.084 | train_step_timing in s: 2.304 | consumed_samples: 6848 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 214/999 | lr: 9.283e-05 | global_batch_size: 32 | global_step: 214 | reduced_train_loss: 1.262 | train_step_timing in s: 2.344 | consumed_samples: 6880 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 215/999 | lr: 9.274e-05 | global_batch_size: 32 | global_step: 215 | reduced_train_loss: 1.127 | train_step_timing in s: 2.336 | consumed_samples: 6912 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 216/999 | lr: 9.265e-05 | global_batch_size: 32 | global_step: 216 | reduced_train_loss: 1.234 | train_step_timing in s: 2.443 | consumed_samples: 6944 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 217/999 | lr: 9.257e-05 | global_batch_size: 32 | global_step: 217 | reduced_train_loss: 1.001 | train_step_timing in s: 2.369 | consumed_samples: 6976 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 218/999 | lr: 9.248e-05 | global_batch_size: 32 | global_step: 218 | reduced_train_loss: 1.196 | train_step_timing in s: 2.254 | consumed_samples: 7008 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 219/999 | lr: 9.239e-05 | global_batch_size: 32 | global_step: 219 | reduced_train_loss: 1.106 | train_step_timing in s: 2.361 | consumed_samples: 7040 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 220/999 | lr: 9.23e-05 | global_batch_size: 32 | global_step: 220 | reduced_train_loss: 1.231 | train_step_timing in s: 2.412 | consumed_samples: 7072 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 221/999 | lr: 9.222e-05 | global_batch_size: 32 | global_step: 221 | reduced_train_loss: 1.092 | train_step_timing in s: 2.384 | consumed_samples: 7104 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 222/999 | lr: 9.213e-05 | global_batch_size: 32 | global_step: 222 | reduced_train_loss: 1.182 | train_step_timing in s: 2.374 | consumed_samples: 7136 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 223/999 | lr: 9.204e-05 | global_batch_size: 32 | global_step: 223 | reduced_train_loss: 1.232 | train_step_timing in s: 2.325 | consumed_samples: 7168 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 224/999 | lr: 9.195e-05 | global_batch_size: 32 | global_step: 224 | reduced_train_loss: 1.085 | train_step_timing in s: 2.309 | consumed_samples: 7200 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 225/999 | lr: 9.186e-05 | global_batch_size: 32 | global_step: 225 | reduced_train_loss: 1.22 | train_step_timing in s: 2.267 | consumed_samples: 7232 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 226/999 | lr: 9.177e-05 | global_batch_size: 32 | global_step: 226 | reduced_train_loss: 1.174 | train_step_timing in s: 2.394 | consumed_samples: 7264 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 227/999 | lr: 9.168e-05 | global_batch_size: 32 | global_step: 227 | reduced_train_loss: 1.081 | train_step_timing in s: 2.292 | consumed_samples: 7296 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 228/999 | lr: 9.158e-05 | global_batch_size: 32 | global_step: 228 | reduced_train_loss: 1.245 | train_step_timing in s: 2.458 | consumed_samples: 7328 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 229/999 | lr: 9.149e-05 | global_batch_size: 32 | global_step: 229 | reduced_train_loss: 1.094 | train_step_timing in s: 2.391 | consumed_samples: 7360 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 230/999 | lr: 9.14e-05 | global_batch_size: 32 | global_step: 230 | reduced_train_loss: 1.119 | train_step_timing in s: 2.381 | consumed_samples: 7392 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 231/999 | lr: 9.131e-05 | global_batch_size: 32 | global_step: 231 | reduced_train_loss: 1.106 | train_step_timing in s: 2.409 | consumed_samples: 7424 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 232/999 | lr: 9.121e-05 | global_batch_size: 32 | global_step: 232 | reduced_train_loss: 0.9476 | train_step_timing in s: 2.454 | consumed_samples: 7456 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 233/999 | lr: 9.112e-05 | global_batch_size: 32 | global_step: 233 | reduced_train_loss: 1.452 | train_step_timing in s: 2.372 | consumed_samples: 7488 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 234/999 | lr: 9.103e-05 | global_batch_size: 32 | global_step: 234 | reduced_train_loss: 1.178 | train_step_timing in s: 2.391 | consumed_samples: 7520 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 235/999 | lr: 9.093e-05 | global_batch_size: 32 | global_step: 235 | reduced_train_loss: 1.182 | train_step_timing in s: 2.377 | consumed_samples: 7552 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 236/999 | lr: 9.084e-05 | global_batch_size: 32 | global_step: 236 | reduced_train_loss: 1.126 | train_step_timing in s: 2.391 | consumed_samples: 7584 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 237/999 | lr: 9.074e-05 | global_batch_size: 32 | global_step: 237 | reduced_train_loss: 1.114 | train_step_timing in s: 2.371 | consumed_samples: 7616 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 238/999 | lr: 9.064e-05 | global_batch_size: 32 | global_step: 238 | reduced_train_loss: 1.213 | train_step_timing in s: 2.367 | consumed_samples: 7648 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 239/999 | lr: 9.055e-05 | global_batch_size: 32 | global_step: 239 | reduced_train_loss: 1.056 | train_step_timing in s: 2.288 | consumed_samples: 7680 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:14:54 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 240 : Start time: 1764080094.027s : Save duration: 0.169s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:14:54 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=239-consumed_samples=7680.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 240/999 | lr: 9.045e-05 | global_batch_size: 32 | global_step: 240 | reduced_train_loss: 1.238 | train_step_timing in s: 3.006 | consumed_samples: 7712 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:14:57 pl:222] Successfully saved checkpoint from iteration     240 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=239-consumed_samples=7680.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:14:57 model_checkpoint:657] Async checkpoint save for step 240 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=239-consumed_samples=7680.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:14:57 dist_ckpt_io:160] Async finalization time took 0.116 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 241/999 | lr: 9.035e-05 | global_batch_size: 32 | global_step: 241 | reduced_train_loss: 1.176 | train_step_timing in s: 2.376 | consumed_samples: 7744 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 242/999 | lr: 9.026e-05 | global_batch_size: 32 | global_step: 242 | reduced_train_loss: 1.256 | train_step_timing in s: 2.41 | consumed_samples: 7776 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 243/999 | lr: 9.016e-05 | global_batch_size: 32 | global_step: 243 | reduced_train_loss: 1.107 | train_step_timing in s: 2.389 | consumed_samples: 7808 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 244/999 | lr: 9.006e-05 | global_batch_size: 32 | global_step: 244 | reduced_train_loss: 1.185 | train_step_timing in s: 2.4 | consumed_samples: 7840 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 245/999 | lr: 8.996e-05 | global_batch_size: 32 | global_step: 245 | reduced_train_loss: 1.278 | train_step_timing in s: 2.269 | consumed_samples: 7872 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 246/999 | lr: 8.986e-05 | global_batch_size: 32 | global_step: 246 | reduced_train_loss: 1.076 | train_step_timing in s: 2.335 | consumed_samples: 7904 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 247/999 | lr: 8.976e-05 | global_batch_size: 32 | global_step: 247 | reduced_train_loss: 1.149 | train_step_timing in s: 2.392 | consumed_samples: 7936 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 248/999 | lr: 8.966e-05 | global_batch_size: 32 | global_step: 248 | reduced_train_loss: 1.319 | train_step_timing in s: 2.394 | consumed_samples: 7968 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 249: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:Training epoch 0, iteration 249/999 | lr: 8.956e-05 | global_batch_size: 32 | global_step: 249 | reduced_train_loss: 1.162 | train_step_timing in s: 2.403 | consumed_samples: 8000 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:15:20 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 249 : Start time: 1764080119.263s : Save duration: 0.771s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:15:20 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=249-consumed_samples=8000.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:15:20 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 250/999 | lr: 8.946e-05 | global_batch_size: 32 | global_step: 250 | reduced_train_loss: 1.091 | train_step_timing in s: 2.821 | consumed_samples: 8032 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:15:23 pl:222] Successfully saved checkpoint from iteration     249 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=249-consumed_samples=8000.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:15:23 model_checkpoint:657] Async checkpoint save for step 250 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=249-consumed_samples=8000.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:15:23 dist_ckpt_io:160] Async finalization time took 0.114 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 251/999 | lr: 8.936e-05 | global_batch_size: 32 | global_step: 251 | reduced_train_loss: 1.174 | train_step_timing in s: 2.427 | consumed_samples: 8064 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 252/999 | lr: 8.925e-05 | global_batch_size: 32 | global_step: 252 | reduced_train_loss: 1.057 | train_step_timing in s: 2.502 | consumed_samples: 8096 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 253/999 | lr: 8.915e-05 | global_batch_size: 32 | global_step: 253 | reduced_train_loss: 1.235 | train_step_timing in s: 2.363 | consumed_samples: 8128 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 254/999 | lr: 8.905e-05 | global_batch_size: 32 | global_step: 254 | reduced_train_loss: 1.193 | train_step_timing in s: 2.458 | consumed_samples: 8160 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 255/999 | lr: 8.894e-05 | global_batch_size: 32 | global_step: 255 | reduced_train_loss: 1.044 | train_step_timing in s: 2.528 | consumed_samples: 8192 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 256/999 | lr: 8.884e-05 | global_batch_size: 32 | global_step: 256 | reduced_train_loss: 1.075 | train_step_timing in s: 2.467 | consumed_samples: 8224 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 257/999 | lr: 8.874e-05 | global_batch_size: 32 | global_step: 257 | reduced_train_loss: 1.36 | train_step_timing in s: 2.381 | consumed_samples: 8256 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 258/999 | lr: 8.863e-05 | global_batch_size: 32 | global_step: 258 | reduced_train_loss: 1.101 | train_step_timing in s: 2.338 | consumed_samples: 8288 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 259/999 | lr: 8.853e-05 | global_batch_size: 32 | global_step: 259 | reduced_train_loss: 1.146 | train_step_timing in s: 2.271 | consumed_samples: 8320 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 260/999 | lr: 8.842e-05 | global_batch_size: 32 | global_step: 260 | reduced_train_loss: 0.9883 | train_step_timing in s: 2.515 | consumed_samples: 8352 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 261/999 | lr: 8.831e-05 | global_batch_size: 32 | global_step: 261 | reduced_train_loss: 1.171 | train_step_timing in s: 2.436 | consumed_samples: 8384 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 262/999 | lr: 8.821e-05 | global_batch_size: 32 | global_step: 262 | reduced_train_loss: 1.142 | train_step_timing in s: 2.401 | consumed_samples: 8416 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 263/999 | lr: 8.81e-05 | global_batch_size: 32 | global_step: 263 | reduced_train_loss: 1.182 | train_step_timing in s: 2.387 | consumed_samples: 8448 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 264/999 | lr: 8.799e-05 | global_batch_size: 32 | global_step: 264 | reduced_train_loss: 1.035 | train_step_timing in s: 2.388 | consumed_samples: 8480 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 265/999 | lr: 8.789e-05 | global_batch_size: 32 | global_step: 265 | reduced_train_loss: 1.164 | train_step_timing in s: 2.513 | consumed_samples: 8512 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 266/999 | lr: 8.778e-05 | global_batch_size: 32 | global_step: 266 | reduced_train_loss: 1.137 | train_step_timing in s: 2.397 | consumed_samples: 8544 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 267/999 | lr: 8.767e-05 | global_batch_size: 32 | global_step: 267 | reduced_train_loss: 1.287 | train_step_timing in s: 2.384 | consumed_samples: 8576 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 268/999 | lr: 8.756e-05 | global_batch_size: 32 | global_step: 268 | reduced_train_loss: 0.9896 | train_step_timing in s: 2.428 | consumed_samples: 8608 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 269/999 | lr: 8.745e-05 | global_batch_size: 32 | global_step: 269 | reduced_train_loss: 1.233 | train_step_timing in s: 2.518 | consumed_samples: 8640 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:17:06 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 270 : Start time: 1764080226.603s : Save duration: 0.203s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:17:07 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=269-consumed_samples=8640.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 270/999 | lr: 8.734e-05 | global_batch_size: 32 | global_step: 270 | reduced_train_loss: 1.131 | train_step_timing in s: 2.874 | consumed_samples: 8672 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:17:10 pl:222] Successfully saved checkpoint from iteration     270 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=269-consumed_samples=8640.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:17:10 model_checkpoint:657] Async checkpoint save for step 270 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=269-consumed_samples=8640.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:17:10 dist_ckpt_io:160] Async finalization time took 0.111 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 271/999 | lr: 8.723e-05 | global_batch_size: 32 | global_step: 271 | reduced_train_loss: 1.286 | train_step_timing in s: 2.381 | consumed_samples: 8704 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 272/999 | lr: 8.712e-05 | global_batch_size: 32 | global_step: 272 | reduced_train_loss: 1.178 | train_step_timing in s: 2.379 | consumed_samples: 8736 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 273/999 | lr: 8.701e-05 | global_batch_size: 32 | global_step: 273 | reduced_train_loss: 1.087 | train_step_timing in s: 2.412 | consumed_samples: 8768 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 274/999 | lr: 8.69e-05 | global_batch_size: 32 | global_step: 274 | reduced_train_loss: 1.005 | train_step_timing in s: 2.369 | consumed_samples: 8800 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 275/999 | lr: 8.679e-05 | global_batch_size: 32 | global_step: 275 | reduced_train_loss: 1.286 | train_step_timing in s: 2.414 | consumed_samples: 8832 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 276/999 | lr: 8.667e-05 | global_batch_size: 32 | global_step: 276 | reduced_train_loss: 1.146 | train_step_timing in s: 2.41 | consumed_samples: 8864 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 277/999 | lr: 8.656e-05 | global_batch_size: 32 | global_step: 277 | reduced_train_loss: 1.063 | train_step_timing in s: 2.619 | consumed_samples: 8896 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 278/999 | lr: 8.645e-05 | global_batch_size: 32 | global_step: 278 | reduced_train_loss: 1.176 | train_step_timing in s: 2.349 | consumed_samples: 8928 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 279/999 | lr: 8.634e-05 | global_batch_size: 32 | global_step: 279 | reduced_train_loss: 1.113 | train_step_timing in s: 2.331 | consumed_samples: 8960 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 280/999 | lr: 8.622e-05 | global_batch_size: 32 | global_step: 280 | reduced_train_loss: 1.032 | train_step_timing in s: 2.419 | consumed_samples: 8992 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 281/999 | lr: 8.611e-05 | global_batch_size: 32 | global_step: 281 | reduced_train_loss: 1.026 | train_step_timing in s: 2.378 | consumed_samples: 9024 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 282/999 | lr: 8.599e-05 | global_batch_size: 32 | global_step: 282 | reduced_train_loss: 1.205 | train_step_timing in s: 2.312 | consumed_samples: 9056 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 283/999 | lr: 8.588e-05 | global_batch_size: 32 | global_step: 283 | reduced_train_loss: 1.212 | train_step_timing in s: 2.426 | consumed_samples: 9088 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 284/999 | lr: 8.576e-05 | global_batch_size: 32 | global_step: 284 | reduced_train_loss: 1.1 | train_step_timing in s: 2.427 | consumed_samples: 9120 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 285/999 | lr: 8.565e-05 | global_batch_size: 32 | global_step: 285 | reduced_train_loss: 1.05 | train_step_timing in s: 2.497 | consumed_samples: 9152 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 286/999 | lr: 8.553e-05 | global_batch_size: 32 | global_step: 286 | reduced_train_loss: 1.086 | train_step_timing in s: 2.398 | consumed_samples: 9184 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 287/999 | lr: 8.541e-05 | global_batch_size: 32 | global_step: 287 | reduced_train_loss: 1.123 | train_step_timing in s: 2.448 | consumed_samples: 9216 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 288/999 | lr: 8.53e-05 | global_batch_size: 32 | global_step: 288 | reduced_train_loss: 1.193 | train_step_timing in s: 2.412 | consumed_samples: 9248 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 289/999 | lr: 8.518e-05 | global_batch_size: 32 | global_step: 289 | reduced_train_loss: 1.084 | train_step_timing in s: 2.433 | consumed_samples: 9280 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 290/999 | lr: 8.506e-05 | global_batch_size: 32 | global_step: 290 | reduced_train_loss: 1.204 | train_step_timing in s: 2.434 | consumed_samples: 9312 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 291/999 | lr: 8.494e-05 | global_batch_size: 32 | global_step: 291 | reduced_train_loss: 0.9336 | train_step_timing in s: 2.353 | consumed_samples: 9344 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 292/999 | lr: 8.483e-05 | global_batch_size: 32 | global_step: 292 | reduced_train_loss: 1.076 | train_step_timing in s: 2.38 | consumed_samples: 9376 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 293/999 | lr: 8.471e-05 | global_batch_size: 32 | global_step: 293 | reduced_train_loss: 1.148 | train_step_timing in s: 2.43 | consumed_samples: 9408 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 294/999 | lr: 8.459e-05 | global_batch_size: 32 | global_step: 294 | reduced_train_loss: 1.048 | train_step_timing in s: 2.419 | consumed_samples: 9440 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 295/999 | lr: 8.447e-05 | global_batch_size: 32 | global_step: 295 | reduced_train_loss: 1.104 | train_step_timing in s: 2.294 | consumed_samples: 9472 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 296/999 | lr: 8.435e-05 | global_batch_size: 32 | global_step: 296 | reduced_train_loss: 1.177 | train_step_timing in s: 2.387 | consumed_samples: 9504 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 297/999 | lr: 8.423e-05 | global_batch_size: 32 | global_step: 297 | reduced_train_loss: 1.282 | train_step_timing in s: 2.35 | consumed_samples: 9536 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 298/999 | lr: 8.411e-05 | global_batch_size: 32 | global_step: 298 | reduced_train_loss: 1.041 | train_step_timing in s: 2.446 | consumed_samples: 9568 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 299/999 | lr: 8.399e-05 | global_batch_size: 32 | global_step: 299 | reduced_train_loss: 0.9998 | train_step_timing in s: 2.471 | consumed_samples: 9600 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 299: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:18:20 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 299 : Start time: 1764080300.559s : Save duration: 0.170s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:18:21 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=299-consumed_samples=9600.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:18:21 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:Training epoch 0, iteration 300/999 | lr: 8.386e-05 | global_batch_size: 32 | global_step: 300 | reduced_train_loss: 1.302 | train_step_timing in s: 2.321 | consumed_samples: 9632 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:19:20 pl:222] Successfully saved checkpoint from iteration     299 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=299-consumed_samples=9600.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:19:20 model_checkpoint:657] Async checkpoint save for step 300 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=299-consumed_samples=9600.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:19:20 dist_ckpt_io:160] Async finalization time took 0.123 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 301/999 | lr: 8.374e-05 | global_batch_size: 32 | global_step: 301 | reduced_train_loss: 1.301 | train_step_timing in s: 2.342 | consumed_samples: 9664 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 302/999 | lr: 8.362e-05 | global_batch_size: 32 | global_step: 302 | reduced_train_loss: 1.137 | train_step_timing in s: 2.367 | consumed_samples: 9696 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 303/999 | lr: 8.35e-05 | global_batch_size: 32 | global_step: 303 | reduced_train_loss: 1.36 | train_step_timing in s: 2.326 | consumed_samples: 9728 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 304/999 | lr: 8.337e-05 | global_batch_size: 32 | global_step: 304 | reduced_train_loss: 1.106 | train_step_timing in s: 2.45 | consumed_samples: 9760 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 305/999 | lr: 8.325e-05 | global_batch_size: 32 | global_step: 305 | reduced_train_loss: 1.055 | train_step_timing in s: 2.372 | consumed_samples: 9792 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 306/999 | lr: 8.313e-05 | global_batch_size: 32 | global_step: 306 | reduced_train_loss: 1.096 | train_step_timing in s: 2.425 | consumed_samples: 9824 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 307/999 | lr: 8.3e-05 | global_batch_size: 32 | global_step: 307 | reduced_train_loss: 1.157 | train_step_timing in s: 2.408 | consumed_samples: 9856 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 308/999 | lr: 8.288e-05 | global_batch_size: 32 | global_step: 308 | reduced_train_loss: 1.183 | train_step_timing in s: 2.432 | consumed_samples: 9888 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 309/999 | lr: 8.275e-05 | global_batch_size: 32 | global_step: 309 | reduced_train_loss: 1.202 | train_step_timing in s: 2.431 | consumed_samples: 9920 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 310/999 | lr: 8.263e-05 | global_batch_size: 32 | global_step: 310 | reduced_train_loss: 1.055 | train_step_timing in s: 2.479 | consumed_samples: 9952 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 311/999 | lr: 8.25e-05 | global_batch_size: 32 | global_step: 311 | reduced_train_loss: 1.12 | train_step_timing in s: 2.358 | consumed_samples: 9984 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 312/999 | lr: 8.238e-05 | global_batch_size: 32 | global_step: 312 | reduced_train_loss: 1.137 | train_step_timing in s: 2.342 | consumed_samples: 10016 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 313/999 | lr: 8.225e-05 | global_batch_size: 32 | global_step: 313 | reduced_train_loss: 1.034 | train_step_timing in s: 2.369 | consumed_samples: 10048 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 314/999 | lr: 8.213e-05 | global_batch_size: 32 | global_step: 314 | reduced_train_loss: 1.311 | train_step_timing in s: 2.37 | consumed_samples: 10080 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 315/999 | lr: 8.2e-05 | global_batch_size: 32 | global_step: 315 | reduced_train_loss: 1.108 | train_step_timing in s: 2.385 | consumed_samples: 10112 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 316/999 | lr: 8.187e-05 | global_batch_size: 32 | global_step: 316 | reduced_train_loss: 1.019 | train_step_timing in s: 2.263 | consumed_samples: 10144 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 317/999 | lr: 8.174e-05 | global_batch_size: 32 | global_step: 317 | reduced_train_loss: 1.038 | train_step_timing in s: 2.353 | consumed_samples: 10176 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 318/999 | lr: 8.162e-05 | global_batch_size: 32 | global_step: 318 | reduced_train_loss: 1.071 | train_step_timing in s: 2.372 | consumed_samples: 10208 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 319/999 | lr: 8.149e-05 | global_batch_size: 32 | global_step: 319 | reduced_train_loss: 1.145 | train_step_timing in s: 2.397 | consumed_samples: 10240 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 320/999 | lr: 8.136e-05 | global_batch_size: 32 | global_step: 320 | reduced_train_loss: 1.23 | train_step_timing in s: 2.419 | consumed_samples: 10272 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 321/999 | lr: 8.123e-05 | global_batch_size: 32 | global_step: 321 | reduced_train_loss: 1.175 | train_step_timing in s: 2.431 | consumed_samples: 10304 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 322/999 | lr: 8.11e-05 | global_batch_size: 32 | global_step: 322 | reduced_train_loss: 1.212 | train_step_timing in s: 2.381 | consumed_samples: 10336 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 323/999 | lr: 8.097e-05 | global_batch_size: 32 | global_step: 323 | reduced_train_loss: 1.079 | train_step_timing in s: 2.375 | consumed_samples: 10368 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 324/999 | lr: 8.084e-05 | global_batch_size: 32 | global_step: 324 | reduced_train_loss: 1.138 | train_step_timing in s: 2.388 | consumed_samples: 10400 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 325/999 | lr: 8.071e-05 | global_batch_size: 32 | global_step: 325 | reduced_train_loss: 1.024 | train_step_timing in s: 2.389 | consumed_samples: 10432 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 326/999 | lr: 8.058e-05 | global_batch_size: 32 | global_step: 326 | reduced_train_loss: 1.071 | train_step_timing in s: 2.436 | consumed_samples: 10464 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 327/999 | lr: 8.045e-05 | global_batch_size: 32 | global_step: 327 | reduced_train_loss: 1.144 | train_step_timing in s: 2.411 | consumed_samples: 10496 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 328/999 | lr: 8.032e-05 | global_batch_size: 32 | global_step: 328 | reduced_train_loss: 1.117 | train_step_timing in s: 2.339 | consumed_samples: 10528 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 329/999 | lr: 8.019e-05 | global_batch_size: 32 | global_step: 329 | reduced_train_loss: 1.156 | train_step_timing in s: 2.36 | consumed_samples: 10560 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:21:27 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 330 : Start time: 1764080486.613s : Save duration: 0.677s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:21:27 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=329-consumed_samples=10560.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 330/999 | lr: 8.005e-05 | global_batch_size: 32 | global_step: 330 | reduced_train_loss: 0.9852 | train_step_timing in s: 2.856 | consumed_samples: 10592 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:21:30 pl:222] Successfully saved checkpoint from iteration     330 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=329-consumed_samples=10560.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:21:30 model_checkpoint:657] Async checkpoint save for step 330 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=329-consumed_samples=10560.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:21:30 dist_ckpt_io:160] Async finalization time took 0.112 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 331/999 | lr: 7.992e-05 | global_batch_size: 32 | global_step: 331 | reduced_train_loss: 1.276 | train_step_timing in s: 2.403 | consumed_samples: 10624 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 332/999 | lr: 7.979e-05 | global_batch_size: 32 | global_step: 332 | reduced_train_loss: 0.8864 | train_step_timing in s: 2.408 | consumed_samples: 10656 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 333/999 | lr: 7.966e-05 | global_batch_size: 32 | global_step: 333 | reduced_train_loss: 1.202 | train_step_timing in s: 2.365 | consumed_samples: 10688 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 334/999 | lr: 7.952e-05 | global_batch_size: 32 | global_step: 334 | reduced_train_loss: 1.061 | train_step_timing in s: 2.374 | consumed_samples: 10720 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 335/999 | lr: 7.939e-05 | global_batch_size: 32 | global_step: 335 | reduced_train_loss: 1.256 | train_step_timing in s: 2.385 | consumed_samples: 10752 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 336/999 | lr: 7.926e-05 | global_batch_size: 32 | global_step: 336 | reduced_train_loss: 1.143 | train_step_timing in s: 2.344 | consumed_samples: 10784 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 337/999 | lr: 7.912e-05 | global_batch_size: 32 | global_step: 337 | reduced_train_loss: 1.076 | train_step_timing in s: 2.372 | consumed_samples: 10816 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 338/999 | lr: 7.899e-05 | global_batch_size: 32 | global_step: 338 | reduced_train_loss: 1.315 | train_step_timing in s: 2.285 | consumed_samples: 10848 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 339/999 | lr: 7.885e-05 | global_batch_size: 32 | global_step: 339 | reduced_train_loss: 0.8567 | train_step_timing in s: 2.411 | consumed_samples: 10880 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 340/999 | lr: 7.872e-05 | global_batch_size: 32 | global_step: 340 | reduced_train_loss: 1.005 | train_step_timing in s: 2.362 | consumed_samples: 10912 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 341/999 | lr: 7.858e-05 | global_batch_size: 32 | global_step: 341 | reduced_train_loss: 1.013 | train_step_timing in s: 2.461 | consumed_samples: 10944 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 342/999 | lr: 7.845e-05 | global_batch_size: 32 | global_step: 342 | reduced_train_loss: 0.9854 | train_step_timing in s: 2.442 | consumed_samples: 10976 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 343/999 | lr: 7.831e-05 | global_batch_size: 32 | global_step: 343 | reduced_train_loss: 1.242 | train_step_timing in s: 2.4 | consumed_samples: 11008 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 344/999 | lr: 7.817e-05 | global_batch_size: 32 | global_step: 344 | reduced_train_loss: 1.217 | train_step_timing in s: 2.377 | consumed_samples: 11040 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 345/999 | lr: 7.804e-05 | global_batch_size: 32 | global_step: 345 | reduced_train_loss: 1.075 | train_step_timing in s: 2.358 | consumed_samples: 11072 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 346/999 | lr: 7.79e-05 | global_batch_size: 32 | global_step: 346 | reduced_train_loss: 0.9887 | train_step_timing in s: 2.444 | consumed_samples: 11104 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 347/999 | lr: 7.776e-05 | global_batch_size: 32 | global_step: 347 | reduced_train_loss: 1.178 | train_step_timing in s: 2.405 | consumed_samples: 11136 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 348/999 | lr: 7.762e-05 | global_batch_size: 32 | global_step: 348 | reduced_train_loss: 1.237 | train_step_timing in s: 2.427 | consumed_samples: 11168 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 349/999 | lr: 7.749e-05 | global_batch_size: 32 | global_step: 349 | reduced_train_loss: 1.292 | train_step_timing in s: 2.338 | consumed_samples: 11200 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 349: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:22:16 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 349 : Start time: 1764080536.320s : Save duration: 0.167s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:22:16 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=349-consumed_samples=11200.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:22:16 dist_ckpt_io:160] Async finalization time took 0.002 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 350/999 | lr: 7.735e-05 | global_batch_size: 32 | global_step: 350 | reduced_train_loss: 1.158 | train_step_timing in s: 2.858 | consumed_samples: 11232 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:22:19 pl:222] Successfully saved checkpoint from iteration     349 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=349-consumed_samples=11200.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:22:19 model_checkpoint:657] Async checkpoint save for step 350 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=349-consumed_samples=11200.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:22:19 dist_ckpt_io:160] Async finalization time took 0.113 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 351/999 | lr: 7.721e-05 | global_batch_size: 32 | global_step: 351 | reduced_train_loss: 1.099 | train_step_timing in s: 2.498 | consumed_samples: 11264 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 352/999 | lr: 7.707e-05 | global_batch_size: 32 | global_step: 352 | reduced_train_loss: 1.088 | train_step_timing in s: 2.439 | consumed_samples: 11296 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 353/999 | lr: 7.693e-05 | global_batch_size: 32 | global_step: 353 | reduced_train_loss: 1.066 | train_step_timing in s: 2.32 | consumed_samples: 11328 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 354/999 | lr: 7.679e-05 | global_batch_size: 32 | global_step: 354 | reduced_train_loss: 1.176 | train_step_timing in s: 2.247 | consumed_samples: 11360 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 355/999 | lr: 7.665e-05 | global_batch_size: 32 | global_step: 355 | reduced_train_loss: 1.253 | train_step_timing in s: 2.357 | consumed_samples: 11392 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 356/999 | lr: 7.651e-05 | global_batch_size: 32 | global_step: 356 | reduced_train_loss: 1.058 | train_step_timing in s: 2.406 | consumed_samples: 11424 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 357/999 | lr: 7.637e-05 | global_batch_size: 32 | global_step: 357 | reduced_train_loss: 1.045 | train_step_timing in s: 2.406 | consumed_samples: 11456 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 358/999 | lr: 7.623e-05 | global_batch_size: 32 | global_step: 358 | reduced_train_loss: 1.007 | train_step_timing in s: 2.384 | consumed_samples: 11488 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 359/999 | lr: 7.609e-05 | global_batch_size: 32 | global_step: 359 | reduced_train_loss: 1.079 | train_step_timing in s: 2.34 | consumed_samples: 11520 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:23:37 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 360 : Start time: 1764080617.416s : Save duration: 0.169s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:23:37 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=359-consumed_samples=11520.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 360/999 | lr: 7.595e-05 | global_batch_size: 32 | global_step: 360 | reduced_train_loss: 1.191 | train_step_timing in s: 2.866 | consumed_samples: 11552 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:23:40 pl:222] Successfully saved checkpoint from iteration     360 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=359-consumed_samples=11520.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:23:40 model_checkpoint:657] Async checkpoint save for step 360 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=359-consumed_samples=11520.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:23:40 dist_ckpt_io:160] Async finalization time took 0.125 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 361/999 | lr: 7.581e-05 | global_batch_size: 32 | global_step: 361 | reduced_train_loss: 1.108 | train_step_timing in s: 2.396 | consumed_samples: 11584 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 362/999 | lr: 7.567e-05 | global_batch_size: 32 | global_step: 362 | reduced_train_loss: 1.135 | train_step_timing in s: 2.433 | consumed_samples: 11616 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 363/999 | lr: 7.552e-05 | global_batch_size: 32 | global_step: 363 | reduced_train_loss: 1.147 | train_step_timing in s: 2.343 | consumed_samples: 11648 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 364/999 | lr: 7.538e-05 | global_batch_size: 32 | global_step: 364 | reduced_train_loss: 1.188 | train_step_timing in s: 2.434 | consumed_samples: 11680 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 365/999 | lr: 7.524e-05 | global_batch_size: 32 | global_step: 365 | reduced_train_loss: 1.15 | train_step_timing in s: 2.416 | consumed_samples: 11712 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 366/999 | lr: 7.51e-05 | global_batch_size: 32 | global_step: 366 | reduced_train_loss: 1.146 | train_step_timing in s: 2.381 | consumed_samples: 11744 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 367/999 | lr: 7.495e-05 | global_batch_size: 32 | global_step: 367 | reduced_train_loss: 1.105 | train_step_timing in s: 2.403 | consumed_samples: 11776 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 368/999 | lr: 7.481e-05 | global_batch_size: 32 | global_step: 368 | reduced_train_loss: 1.051 | train_step_timing in s: 2.464 | consumed_samples: 11808 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 369/999 | lr: 7.467e-05 | global_batch_size: 32 | global_step: 369 | reduced_train_loss: 1.297 | train_step_timing in s: 2.291 | consumed_samples: 11840 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 370/999 | lr: 7.452e-05 | global_batch_size: 32 | global_step: 370 | reduced_train_loss: 1.234 | train_step_timing in s: 2.283 | consumed_samples: 11872 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 371/999 | lr: 7.438e-05 | global_batch_size: 32 | global_step: 371 | reduced_train_loss: 1.158 | train_step_timing in s: 2.364 | consumed_samples: 11904 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 372/999 | lr: 7.423e-05 | global_batch_size: 32 | global_step: 372 | reduced_train_loss: 1.047 | train_step_timing in s: 2.462 | consumed_samples: 11936 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 373/999 | lr: 7.409e-05 | global_batch_size: 32 | global_step: 373 | reduced_train_loss: 1.071 | train_step_timing in s: 2.389 | consumed_samples: 11968 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 374/999 | lr: 7.394e-05 | global_batch_size: 32 | global_step: 374 | reduced_train_loss: 1.035 | train_step_timing in s: 2.417 | consumed_samples: 12000 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 375/999 | lr: 7.38e-05 | global_batch_size: 32 | global_step: 375 | reduced_train_loss: 1.257 | train_step_timing in s: 2.432 | consumed_samples: 12032 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 376/999 | lr: 7.365e-05 | global_batch_size: 32 | global_step: 376 | reduced_train_loss: 1.151 | train_step_timing in s: 2.407 | consumed_samples: 12064 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 377/999 | lr: 7.351e-05 | global_batch_size: 32 | global_step: 377 | reduced_train_loss: 1.161 | train_step_timing in s: 2.408 | consumed_samples: 12096 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 378/999 | lr: 7.336e-05 | global_batch_size: 32 | global_step: 378 | reduced_train_loss: 1.127 | train_step_timing in s: 2.443 | consumed_samples: 12128 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 379/999 | lr: 7.321e-05 | global_batch_size: 32 | global_step: 379 | reduced_train_loss: 1.017 | train_step_timing in s: 2.384 | consumed_samples: 12160 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 380/999 | lr: 7.307e-05 | global_batch_size: 32 | global_step: 380 | reduced_train_loss: 1.08 | train_step_timing in s: 2.329 | consumed_samples: 12192 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 381/999 | lr: 7.292e-05 | global_batch_size: 32 | global_step: 381 | reduced_train_loss: 1.253 | train_step_timing in s: 2.418 | consumed_samples: 12224 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 382/999 | lr: 7.277e-05 | global_batch_size: 32 | global_step: 382 | reduced_train_loss: 1.174 | train_step_timing in s: 2.403 | consumed_samples: 12256 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 383/999 | lr: 7.263e-05 | global_batch_size: 32 | global_step: 383 | reduced_train_loss: 1.24 | train_step_timing in s: 2.385 | consumed_samples: 12288 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 384/999 | lr: 7.248e-05 | global_batch_size: 32 | global_step: 384 | reduced_train_loss: 1.315 | train_step_timing in s: 2.376 | consumed_samples: 12320 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 385/999 | lr: 7.233e-05 | global_batch_size: 32 | global_step: 385 | reduced_train_loss: 1.163 | train_step_timing in s: 2.374 | consumed_samples: 12352 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 386/999 | lr: 7.218e-05 | global_batch_size: 32 | global_step: 386 | reduced_train_loss: 1.068 | train_step_timing in s: 2.345 | consumed_samples: 12384 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 387/999 | lr: 7.203e-05 | global_batch_size: 32 | global_step: 387 | reduced_train_loss: 1.28 | train_step_timing in s: 2.365 | consumed_samples: 12416 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 388/999 | lr: 7.189e-05 | global_batch_size: 32 | global_step: 388 | reduced_train_loss: 1.325 | train_step_timing in s: 2.471 | consumed_samples: 12448 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 389/999 | lr: 7.174e-05 | global_batch_size: 32 | global_step: 389 | reduced_train_loss: 1.283 | train_step_timing in s: 2.487 | consumed_samples: 12480 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:25:47 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 390 : Start time: 1764080747.570s : Save duration: 0.180s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:25:48 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=389-consumed_samples=12480.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 390/999 | lr: 7.159e-05 | global_batch_size: 32 | global_step: 390 | reduced_train_loss: 1.04 | train_step_timing in s: 2.822 | consumed_samples: 12512 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:25:50 pl:222] Successfully saved checkpoint from iteration     390 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=389-consumed_samples=12480.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:25:50 model_checkpoint:657] Async checkpoint save for step 390 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=389-consumed_samples=12480.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:25:51 dist_ckpt_io:160] Async finalization time took 0.117 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 391/999 | lr: 7.144e-05 | global_batch_size: 32 | global_step: 391 | reduced_train_loss: 0.9258 | train_step_timing in s: 2.374 | consumed_samples: 12544 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 392/999 | lr: 7.129e-05 | global_batch_size: 32 | global_step: 392 | reduced_train_loss: 1.168 | train_step_timing in s: 2.38 | consumed_samples: 12576 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 393/999 | lr: 7.114e-05 | global_batch_size: 32 | global_step: 393 | reduced_train_loss: 1.176 | train_step_timing in s: 2.379 | consumed_samples: 12608 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 394/999 | lr: 7.099e-05 | global_batch_size: 32 | global_step: 394 | reduced_train_loss: 1.006 | train_step_timing in s: 2.459 | consumed_samples: 12640 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 395/999 | lr: 7.084e-05 | global_batch_size: 32 | global_step: 395 | reduced_train_loss: 1.23 | train_step_timing in s: 2.386 | consumed_samples: 12672 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 396/999 | lr: 7.069e-05 | global_batch_size: 32 | global_step: 396 | reduced_train_loss: 1.23 | train_step_timing in s: 2.342 | consumed_samples: 12704 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 397/999 | lr: 7.054e-05 | global_batch_size: 32 | global_step: 397 | reduced_train_loss: 1.07 | train_step_timing in s: 2.403 | consumed_samples: 12736 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 398/999 | lr: 7.039e-05 | global_batch_size: 32 | global_step: 398 | reduced_train_loss: 1.11 | train_step_timing in s: 2.384 | consumed_samples: 12768 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 399/999 | lr: 7.024e-05 | global_batch_size: 32 | global_step: 399 | reduced_train_loss: 0.9608 | train_step_timing in s: 2.356 | consumed_samples: 12800 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 399: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:26:13 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 399 : Start time: 1764080772.689s : Save duration: 0.587s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:26:13 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=399-consumed_samples=12800.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:26:13 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 400/999 | lr: 7.008e-05 | global_batch_size: 32 | global_step: 400 | reduced_train_loss: 1.297 | train_step_timing in s: 2.938 | consumed_samples: 12832 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:26:16 pl:222] Successfully saved checkpoint from iteration     399 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=399-consumed_samples=12800.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:26:16 model_checkpoint:657] Async checkpoint save for step 400 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=399-consumed_samples=12800.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:26:16 dist_ckpt_io:160] Async finalization time took 0.108 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 401/999 | lr: 6.993e-05 | global_batch_size: 32 | global_step: 401 | reduced_train_loss: 1.075 | train_step_timing in s: 2.35 | consumed_samples: 12864 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 402/999 | lr: 6.978e-05 | global_batch_size: 32 | global_step: 402 | reduced_train_loss: 1.323 | train_step_timing in s: 2.333 | consumed_samples: 12896 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 403/999 | lr: 6.963e-05 | global_batch_size: 32 | global_step: 403 | reduced_train_loss: 1.08 | train_step_timing in s: 2.423 | consumed_samples: 12928 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 404/999 | lr: 6.948e-05 | global_batch_size: 32 | global_step: 404 | reduced_train_loss: 1.173 | train_step_timing in s: 2.372 | consumed_samples: 12960 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 405/999 | lr: 6.932e-05 | global_batch_size: 32 | global_step: 405 | reduced_train_loss: 1.316 | train_step_timing in s: 2.344 | consumed_samples: 12992 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 406/999 | lr: 6.917e-05 | global_batch_size: 32 | global_step: 406 | reduced_train_loss: 1.213 | train_step_timing in s: 2.37 | consumed_samples: 13024 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 407/999 | lr: 6.902e-05 | global_batch_size: 32 | global_step: 407 | reduced_train_loss: 1.011 | train_step_timing in s: 2.389 | consumed_samples: 13056 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 408/999 | lr: 6.887e-05 | global_batch_size: 32 | global_step: 408 | reduced_train_loss: 1.098 | train_step_timing in s: 2.39 | consumed_samples: 13088 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 409/999 | lr: 6.871e-05 | global_batch_size: 32 | global_step: 409 | reduced_train_loss: 1.13 | train_step_timing in s: 2.393 | consumed_samples: 13120 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 410/999 | lr: 6.856e-05 | global_batch_size: 32 | global_step: 410 | reduced_train_loss: 0.9682 | train_step_timing in s: 2.379 | consumed_samples: 13152 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 411/999 | lr: 6.841e-05 | global_batch_size: 32 | global_step: 411 | reduced_train_loss: 1.163 | train_step_timing in s: 2.281 | consumed_samples: 13184 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 412/999 | lr: 6.825e-05 | global_batch_size: 32 | global_step: 412 | reduced_train_loss: 0.9291 | train_step_timing in s: 2.418 | consumed_samples: 13216 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 413/999 | lr: 6.81e-05 | global_batch_size: 32 | global_step: 413 | reduced_train_loss: 1.103 | train_step_timing in s: 2.365 | consumed_samples: 13248 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 414/999 | lr: 6.794e-05 | global_batch_size: 32 | global_step: 414 | reduced_train_loss: 1.456 | train_step_timing in s: 2.322 | consumed_samples: 13280 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 415/999 | lr: 6.779e-05 | global_batch_size: 32 | global_step: 415 | reduced_train_loss: 1.144 | train_step_timing in s: 2.395 | consumed_samples: 13312 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 416/999 | lr: 6.764e-05 | global_batch_size: 32 | global_step: 416 | reduced_train_loss: 1.169 | train_step_timing in s: 2.365 | consumed_samples: 13344 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 417/999 | lr: 6.748e-05 | global_batch_size: 32 | global_step: 417 | reduced_train_loss: 0.9679 | train_step_timing in s: 2.443 | consumed_samples: 13376 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 418/999 | lr: 6.733e-05 | global_batch_size: 32 | global_step: 418 | reduced_train_loss: 1.145 | train_step_timing in s: 2.367 | consumed_samples: 13408 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 419/999 | lr: 6.717e-05 | global_batch_size: 32 | global_step: 419 | reduced_train_loss: 1.093 | train_step_timing in s: 2.404 | consumed_samples: 13440 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0[default0]:
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:27:58 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 420 : Start time: 1764080878.773s : Save duration: 0.172s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:27:59 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=419-consumed_samples=13440.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 420/999 | lr: 6.701e-05 | global_batch_size: 32 | global_step: 420 | reduced_train_loss: 1.084 | train_step_timing in s: 2.885 | consumed_samples: 13472 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:28:02 pl:222] Successfully saved checkpoint from iteration     420 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=419-consumed_samples=13440.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:28:02 model_checkpoint:657] Async checkpoint save for step 420 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=419-consumed_samples=13440.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:28:02 dist_ckpt_io:160] Async finalization time took 0.113 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 421/999 | lr: 6.686e-05 | global_batch_size: 32 | global_step: 421 | reduced_train_loss: 1.163 | train_step_timing in s: 2.363 | consumed_samples: 13504 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 422/999 | lr: 6.67e-05 | global_batch_size: 32 | global_step: 422 | reduced_train_loss: 1.041 | train_step_timing in s: 2.439 | consumed_samples: 13536 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 423/999 | lr: 6.655e-05 | global_batch_size: 32 | global_step: 423 | reduced_train_loss: 1.1 | train_step_timing in s: 2.386 | consumed_samples: 13568 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 424/999 | lr: 6.639e-05 | global_batch_size: 32 | global_step: 424 | reduced_train_loss: 1.09 | train_step_timing in s: 2.465 | consumed_samples: 13600 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 425/999 | lr: 6.623e-05 | global_batch_size: 32 | global_step: 425 | reduced_train_loss: 1.191 | train_step_timing in s: 2.393 | consumed_samples: 13632 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 426/999 | lr: 6.608e-05 | global_batch_size: 32 | global_step: 426 | reduced_train_loss: 0.9544 | train_step_timing in s: 2.332 | consumed_samples: 13664 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 427/999 | lr: 6.592e-05 | global_batch_size: 32 | global_step: 427 | reduced_train_loss: 1.214 | train_step_timing in s: 2.357 | consumed_samples: 13696 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 428/999 | lr: 6.577e-05 | global_batch_size: 32 | global_step: 428 | reduced_train_loss: 1.138 | train_step_timing in s: 2.374 | consumed_samples: 13728 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 429/999 | lr: 6.561e-05 | global_batch_size: 32 | global_step: 429 | reduced_train_loss: 1.159 | train_step_timing in s: 2.382 | consumed_samples: 13760 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 430/999 | lr: 6.545e-05 | global_batch_size: 32 | global_step: 430 | reduced_train_loss: 1.017 | train_step_timing in s: 2.421 | consumed_samples: 13792 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 431/999 | lr: 6.529e-05 | global_batch_size: 32 | global_step: 431 | reduced_train_loss: 1.161 | train_step_timing in s: 2.412 | consumed_samples: 13824 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 432/999 | lr: 6.514e-05 | global_batch_size: 32 | global_step: 432 | reduced_train_loss: 1.225 | train_step_timing in s: 2.344 | consumed_samples: 13856 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 433/999 | lr: 6.498e-05 | global_batch_size: 32 | global_step: 433 | reduced_train_loss: 1.042 | train_step_timing in s: 2.431 | consumed_samples: 13888 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 434/999 | lr: 6.482e-05 | global_batch_size: 32 | global_step: 434 | reduced_train_loss: 1.219 | train_step_timing in s: 2.399 | consumed_samples: 13920 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 435/999 | lr: 6.466e-05 | global_batch_size: 32 | global_step: 435 | reduced_train_loss: 1.31 | train_step_timing in s: 2.387 | consumed_samples: 13952 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 436/999 | lr: 6.45e-05 | global_batch_size: 32 | global_step: 436 | reduced_train_loss: 1.099 | train_step_timing in s: 2.41 | consumed_samples: 13984 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 437/999 | lr: 6.435e-05 | global_batch_size: 32 | global_step: 437 | reduced_train_loss: 1.25 | train_step_timing in s: 2.355 | consumed_samples: 14016 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 438/999 | lr: 6.419e-05 | global_batch_size: 32 | global_step: 438 | reduced_train_loss: 1.101 | train_step_timing in s: 2.35 | consumed_samples: 14048 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 439/999 | lr: 6.403e-05 | global_batch_size: 32 | global_step: 439 | reduced_train_loss: 1.066 | train_step_timing in s: 2.389 | consumed_samples: 14080 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 440/999 | lr: 6.387e-05 | global_batch_size: 32 | global_step: 440 | reduced_train_loss: 1.248 | train_step_timing in s: 2.348 | consumed_samples: 14112 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 441/999 | lr: 6.371e-05 | global_batch_size: 32 | global_step: 441 | reduced_train_loss: 1.001 | train_step_timing in s: 2.374 | consumed_samples: 14144 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 442/999 | lr: 6.355e-05 | global_batch_size: 32 | global_step: 442 | reduced_train_loss: 1.149 | train_step_timing in s: 2.411 | consumed_samples: 14176 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 443/999 | lr: 6.339e-05 | global_batch_size: 32 | global_step: 443 | reduced_train_loss: 1.087 | train_step_timing in s: 2.392 | consumed_samples: 14208 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 444/999 | lr: 6.323e-05 | global_batch_size: 32 | global_step: 444 | reduced_train_loss: 0.9967 | train_step_timing in s: 2.394 | consumed_samples: 14240 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 445/999 | lr: 6.307e-05 | global_batch_size: 32 | global_step: 445 | reduced_train_loss: 1.255 | train_step_timing in s: 2.36 | consumed_samples: 14272 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 446/999 | lr: 6.291e-05 | global_batch_size: 32 | global_step: 446 | reduced_train_loss: 1.064 | train_step_timing in s: 2.339 | consumed_samples: 14304 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 447/999 | lr: 6.275e-05 | global_batch_size: 32 | global_step: 447 | reduced_train_loss: 1.138 | train_step_timing in s: 2.399 | consumed_samples: 14336 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 448/999 | lr: 6.259e-05 | global_batch_size: 32 | global_step: 448 | reduced_train_loss: 0.988 | train_step_timing in s: 2.469 | consumed_samples: 14368 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 449/999 | lr: 6.243e-05 | global_batch_size: 32 | global_step: 449 | reduced_train_loss: 1.225 | train_step_timing in s: 2.371 | consumed_samples: 14400 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 449: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:29:12 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 449 : Start time: 1764080952.049s : Save duration: 0.172s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:29:12 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=449-consumed_samples=14400.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:29:12 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:Training epoch 0, iteration 450/999 | lr: 6.227e-05 | global_batch_size: 32 | global_step: 450 | reduced_train_loss: 1.001 | train_step_timing in s: 2.447 | consumed_samples: 14432 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:30:12 pl:222] Successfully saved checkpoint from iteration     449 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=449-consumed_samples=14400.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:30:12 model_checkpoint:657] Async checkpoint save for step 450 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=449-consumed_samples=14400.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:30:12 dist_ckpt_io:160] Async finalization time took 0.115 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 451/999 | lr: 6.211e-05 | global_batch_size: 32 | global_step: 451 | reduced_train_loss: 1.089 | train_step_timing in s: 2.447 | consumed_samples: 14464 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 452/999 | lr: 6.195e-05 | global_batch_size: 32 | global_step: 452 | reduced_train_loss: 1.12 | train_step_timing in s: 2.369 | consumed_samples: 14496 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 453/999 | lr: 6.179e-05 | global_batch_size: 32 | global_step: 453 | reduced_train_loss: 1.287 | train_step_timing in s: 2.427 | consumed_samples: 14528 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 454/999 | lr: 6.163e-05 | global_batch_size: 32 | global_step: 454 | reduced_train_loss: 1.185 | train_step_timing in s: 2.43 | consumed_samples: 14560 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 455/999 | lr: 6.147e-05 | global_batch_size: 32 | global_step: 455 | reduced_train_loss: 1.078 | train_step_timing in s: 2.485 | consumed_samples: 14592 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 456/999 | lr: 6.131e-05 | global_batch_size: 32 | global_step: 456 | reduced_train_loss: 1.197 | train_step_timing in s: 2.363 | consumed_samples: 14624 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 457/999 | lr: 6.115e-05 | global_batch_size: 32 | global_step: 457 | reduced_train_loss: 1.369 | train_step_timing in s: 2.372 | consumed_samples: 14656 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 458/999 | lr: 6.099e-05 | global_batch_size: 32 | global_step: 458 | reduced_train_loss: 0.8292 | train_step_timing in s: 2.414 | consumed_samples: 14688 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 459/999 | lr: 6.083e-05 | global_batch_size: 32 | global_step: 459 | reduced_train_loss: 1.035 | train_step_timing in s: 2.452 | consumed_samples: 14720 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 460/999 | lr: 6.066e-05 | global_batch_size: 32 | global_step: 460 | reduced_train_loss: 1.215 | train_step_timing in s: 2.438 | consumed_samples: 14752 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 461/999 | lr: 6.05e-05 | global_batch_size: 32 | global_step: 461 | reduced_train_loss: 0.978 | train_step_timing in s: 2.439 | consumed_samples: 14784 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 462/999 | lr: 6.034e-05 | global_batch_size: 32 | global_step: 462 | reduced_train_loss: 1.209 | train_step_timing in s: 2.328 | consumed_samples: 14816 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 463/999 | lr: 6.018e-05 | global_batch_size: 32 | global_step: 463 | reduced_train_loss: 1.152 | train_step_timing in s: 2.397 | consumed_samples: 14848 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 464/999 | lr: 6.002e-05 | global_batch_size: 32 | global_step: 464 | reduced_train_loss: 0.9743 | train_step_timing in s: 2.377 | consumed_samples: 14880 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 465/999 | lr: 5.986e-05 | global_batch_size: 32 | global_step: 465 | reduced_train_loss: 1.081 | train_step_timing in s: 2.355 | consumed_samples: 14912 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 466/999 | lr: 5.969e-05 | global_batch_size: 32 | global_step: 466 | reduced_train_loss: 1.155 | train_step_timing in s: 2.359 | consumed_samples: 14944 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 467/999 | lr: 5.953e-05 | global_batch_size: 32 | global_step: 467 | reduced_train_loss: 1.18 | train_step_timing in s: 2.4 | consumed_samples: 14976 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 468/999 | lr: 5.937e-05 | global_batch_size: 32 | global_step: 468 | reduced_train_loss: 1.219 | train_step_timing in s: 2.356 | consumed_samples: 15008 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 469/999 | lr: 5.921e-05 | global_batch_size: 32 | global_step: 469 | reduced_train_loss: 1.132 | train_step_timing in s: 2.412 | consumed_samples: 15040 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 470/999 | lr: 5.904e-05 | global_batch_size: 32 | global_step: 470 | reduced_train_loss: 0.9767 | train_step_timing in s: 2.414 | consumed_samples: 15072 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 471/999 | lr: 5.888e-05 | global_batch_size: 32 | global_step: 471 | reduced_train_loss: 1.115 | train_step_timing in s: 2.365 | consumed_samples: 15104 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 472/999 | lr: 5.872e-05 | global_batch_size: 32 | global_step: 472 | reduced_train_loss: 1.278 | train_step_timing in s: 2.34 | consumed_samples: 15136 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 473/999 | lr: 5.856e-05 | global_batch_size: 32 | global_step: 473 | reduced_train_loss: 1.106 | train_step_timing in s: 2.356 | consumed_samples: 15168 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 474/999 | lr: 5.839e-05 | global_batch_size: 32 | global_step: 474 | reduced_train_loss: 1.053 | train_step_timing in s: 2.384 | consumed_samples: 15200 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 475/999 | lr: 5.823e-05 | global_batch_size: 32 | global_step: 475 | reduced_train_loss: 1.205 | train_step_timing in s: 2.303 | consumed_samples: 15232 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 476/999 | lr: 5.807e-05 | global_batch_size: 32 | global_step: 476 | reduced_train_loss: 1.016 | train_step_timing in s: 2.381 | consumed_samples: 15264 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 477/999 | lr: 5.79e-05 | global_batch_size: 32 | global_step: 477 | reduced_train_loss: 1.003 | train_step_timing in s: 2.38 | consumed_samples: 15296 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 478/999 | lr: 5.774e-05 | global_batch_size: 32 | global_step: 478 | reduced_train_loss: 1.005 | train_step_timing in s: 2.419 | consumed_samples: 15328 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 479/999 | lr: 5.758e-05 | global_batch_size: 32 | global_step: 479 | reduced_train_loss: 1.146 | train_step_timing in s: 2.43 | consumed_samples: 15360 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:32:19 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 480 : Start time: 1764081138.896s : Save duration: 0.172s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:32:19 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=479-consumed_samples=15360.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 480/999 | lr: 5.741e-05 | global_batch_size: 32 | global_step: 480 | reduced_train_loss: 1.392 | train_step_timing in s: 2.975 | consumed_samples: 15392 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:32:22 pl:222] Successfully saved checkpoint from iteration     480 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=479-consumed_samples=15360.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:32:22 model_checkpoint:657] Async checkpoint save for step 480 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=479-consumed_samples=15360.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:32:22 dist_ckpt_io:160] Async finalization time took 0.574 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 481/999 | lr: 5.725e-05 | global_batch_size: 32 | global_step: 481 | reduced_train_loss: 1.073 | train_step_timing in s: 2.463 | consumed_samples: 15424 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 482/999 | lr: 5.709e-05 | global_batch_size: 32 | global_step: 482 | reduced_train_loss: 1.022 | train_step_timing in s: 2.439 | consumed_samples: 15456 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 483/999 | lr: 5.692e-05 | global_batch_size: 32 | global_step: 483 | reduced_train_loss: 1.156 | train_step_timing in s: 2.373 | consumed_samples: 15488 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 484/999 | lr: 5.676e-05 | global_batch_size: 32 | global_step: 484 | reduced_train_loss: 1.025 | train_step_timing in s: 2.377 | consumed_samples: 15520 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 485/999 | lr: 5.659e-05 | global_batch_size: 32 | global_step: 485 | reduced_train_loss: 1.177 | train_step_timing in s: 2.427 | consumed_samples: 15552 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 486/999 | lr: 5.643e-05 | global_batch_size: 32 | global_step: 486 | reduced_train_loss: 1.226 | train_step_timing in s: 2.406 | consumed_samples: 15584 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 487/999 | lr: 5.627e-05 | global_batch_size: 32 | global_step: 487 | reduced_train_loss: 1.263 | train_step_timing in s: 2.37 | consumed_samples: 15616 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 488/999 | lr: 5.61e-05 | global_batch_size: 32 | global_step: 488 | reduced_train_loss: 1.161 | train_step_timing in s: 2.373 | consumed_samples: 15648 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 489/999 | lr: 5.594e-05 | global_batch_size: 32 | global_step: 489 | reduced_train_loss: 1.524 | train_step_timing in s: 2.381 | consumed_samples: 15680 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 490/999 | lr: 5.577e-05 | global_batch_size: 32 | global_step: 490 | reduced_train_loss: 1.044 | train_step_timing in s: 2.392 | consumed_samples: 15712 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 491/999 | lr: 5.561e-05 | global_batch_size: 32 | global_step: 491 | reduced_train_loss: 1.153 | train_step_timing in s: 2.416 | consumed_samples: 15744 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 492/999 | lr: 5.545e-05 | global_batch_size: 32 | global_step: 492 | reduced_train_loss: 0.9871 | train_step_timing in s: 2.456 | consumed_samples: 15776 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 493/999 | lr: 5.528e-05 | global_batch_size: 32 | global_step: 493 | reduced_train_loss: 1.15 | train_step_timing in s: 2.364 | consumed_samples: 15808 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 494/999 | lr: 5.512e-05 | global_batch_size: 32 | global_step: 494 | reduced_train_loss: 1.229 | train_step_timing in s: 2.327 | consumed_samples: 15840 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 495/999 | lr: 5.495e-05 | global_batch_size: 32 | global_step: 495 | reduced_train_loss: 1.143 | train_step_timing in s: 2.388 | consumed_samples: 15872 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 496/999 | lr: 5.479e-05 | global_batch_size: 32 | global_step: 496 | reduced_train_loss: 1.174 | train_step_timing in s: 2.438 | consumed_samples: 15904 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 497/999 | lr: 5.462e-05 | global_batch_size: 32 | global_step: 497 | reduced_train_loss: 0.9957 | train_step_timing in s: 2.354 | consumed_samples: 15936 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 498/999 | lr: 5.446e-05 | global_batch_size: 32 | global_step: 498 | reduced_train_loss: 1.198 | train_step_timing in s: 2.38 | consumed_samples: 15968 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 499/999 | lr: 5.429e-05 | global_batch_size: 32 | global_step: 499 | reduced_train_loss: 1.031 | train_step_timing in s: 2.431 | consumed_samples: 16000 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 499: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:33:09 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 499 : Start time: 1764081188.913s : Save duration: 0.171s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:33:09 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=499-consumed_samples=16000.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:33:09 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 500/999 | lr: 5.413e-05 | global_batch_size: 32 | global_step: 500 | reduced_train_loss: 1.145 | train_step_timing in s: 2.783 | consumed_samples: 16032 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:33:12 pl:222] Successfully saved checkpoint from iteration     499 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=499-consumed_samples=16000.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:33:12 model_checkpoint:657] Async checkpoint save for step 500 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=499-consumed_samples=16000.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:33:12 dist_ckpt_io:160] Async finalization time took 0.112 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 501/999 | lr: 5.396e-05 | global_batch_size: 32 | global_step: 501 | reduced_train_loss: 1.007 | train_step_timing in s: 2.367 | consumed_samples: 16064 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 502/999 | lr: 5.38e-05 | global_batch_size: 32 | global_step: 502 | reduced_train_loss: 1.265 | train_step_timing in s: 2.328 | consumed_samples: 16096 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 503/999 | lr: 5.363e-05 | global_batch_size: 32 | global_step: 503 | reduced_train_loss: 1.07 | train_step_timing in s: 2.359 | consumed_samples: 16128 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 504/999 | lr: 5.347e-05 | global_batch_size: 32 | global_step: 504 | reduced_train_loss: 1.015 | train_step_timing in s: 2.34 | consumed_samples: 16160 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 505/999 | lr: 5.33e-05 | global_batch_size: 32 | global_step: 505 | reduced_train_loss: 1.37 | train_step_timing in s: 2.336 | consumed_samples: 16192 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 506/999 | lr: 5.314e-05 | global_batch_size: 32 | global_step: 506 | reduced_train_loss: 1.147 | train_step_timing in s: 2.407 | consumed_samples: 16224 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 507/999 | lr: 5.297e-05 | global_batch_size: 32 | global_step: 507 | reduced_train_loss: 1.254 | train_step_timing in s: 2.341 | consumed_samples: 16256 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 508/999 | lr: 5.281e-05 | global_batch_size: 32 | global_step: 508 | reduced_train_loss: 1.211 | train_step_timing in s: 2.372 | consumed_samples: 16288 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 509/999 | lr: 5.264e-05 | global_batch_size: 32 | global_step: 509 | reduced_train_loss: 1.161 | train_step_timing in s: 2.444 | consumed_samples: 16320 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:34:35 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 510 : Start time: 1764081275.763s : Save duration: 0.168s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:34:36 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=509-consumed_samples=16320.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 510/999 | lr: 5.248e-05 | global_batch_size: 32 | global_step: 510 | reduced_train_loss: 0.9828 | train_step_timing in s: 2.673 | consumed_samples: 16352 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:34:38 pl:222] Successfully saved checkpoint from iteration     510 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=509-consumed_samples=16320.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:34:38 model_checkpoint:657] Async checkpoint save for step 510 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=509-consumed_samples=16320.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:34:39 dist_ckpt_io:160] Async finalization time took 0.114 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 511/999 | lr: 5.231e-05 | global_batch_size: 32 | global_step: 511 | reduced_train_loss: 1.012 | train_step_timing in s: 2.463 | consumed_samples: 16384 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 512/999 | lr: 5.215e-05 | global_batch_size: 32 | global_step: 512 | reduced_train_loss: 1.209 | train_step_timing in s: 2.372 | consumed_samples: 16416 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 513/999 | lr: 5.198e-05 | global_batch_size: 32 | global_step: 513 | reduced_train_loss: 0.9654 | train_step_timing in s: 2.395 | consumed_samples: 16448 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 514/999 | lr: 5.182e-05 | global_batch_size: 32 | global_step: 514 | reduced_train_loss: 0.9965 | train_step_timing in s: 2.36 | consumed_samples: 16480 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 515/999 | lr: 5.165e-05 | global_batch_size: 32 | global_step: 515 | reduced_train_loss: 1.0 | train_step_timing in s: 2.387 | consumed_samples: 16512 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 516/999 | lr: 5.149e-05 | global_batch_size: 32 | global_step: 516 | reduced_train_loss: 1.127 | train_step_timing in s: 2.36 | consumed_samples: 16544 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 517/999 | lr: 5.132e-05 | global_batch_size: 32 | global_step: 517 | reduced_train_loss: 1.168 | train_step_timing in s: 2.421 | consumed_samples: 16576 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 518/999 | lr: 5.116e-05 | global_batch_size: 32 | global_step: 518 | reduced_train_loss: 1.264 | train_step_timing in s: 2.391 | consumed_samples: 16608 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 519/999 | lr: 5.099e-05 | global_batch_size: 32 | global_step: 519 | reduced_train_loss: 1.046 | train_step_timing in s: 2.381 | consumed_samples: 16640 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 520/999 | lr: 5.083e-05 | global_batch_size: 32 | global_step: 520 | reduced_train_loss: 1.258 | train_step_timing in s: 2.34 | consumed_samples: 16672 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 521/999 | lr: 5.066e-05 | global_batch_size: 32 | global_step: 521 | reduced_train_loss: 1.299 | train_step_timing in s: 2.337 | consumed_samples: 16704 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 522/999 | lr: 5.05e-05 | global_batch_size: 32 | global_step: 522 | reduced_train_loss: 0.9791 | train_step_timing in s: 2.425 | consumed_samples: 16736 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 523/999 | lr: 5.033e-05 | global_batch_size: 32 | global_step: 523 | reduced_train_loss: 1.146 | train_step_timing in s: 2.414 | consumed_samples: 16768 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 524/999 | lr: 5.017e-05 | global_batch_size: 32 | global_step: 524 | reduced_train_loss: 0.9233 | train_step_timing in s: 2.422 | consumed_samples: 16800 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 525/999 | lr: 5e-05 | global_batch_size: 32 | global_step: 525 | reduced_train_loss: 1.092 | train_step_timing in s: 2.414 | consumed_samples: 16832 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 526/999 | lr: 4.983e-05 | global_batch_size: 32 | global_step: 526 | reduced_train_loss: 1.363 | train_step_timing in s: 2.377 | consumed_samples: 16864 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 527/999 | lr: 4.967e-05 | global_batch_size: 32 | global_step: 527 | reduced_train_loss: 1.173 | train_step_timing in s: 2.405 | consumed_samples: 16896 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 528/999 | lr: 4.95e-05 | global_batch_size: 32 | global_step: 528 | reduced_train_loss: 1.108 | train_step_timing in s: 2.384 | consumed_samples: 16928 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 529/999 | lr: 4.934e-05 | global_batch_size: 32 | global_step: 529 | reduced_train_loss: 1.1 | train_step_timing in s: 2.387 | consumed_samples: 16960 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 530/999 | lr: 4.917e-05 | global_batch_size: 32 | global_step: 530 | reduced_train_loss: 1.012 | train_step_timing in s: 2.43 | consumed_samples: 16992 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 531/999 | lr: 4.901e-05 | global_batch_size: 32 | global_step: 531 | reduced_train_loss: 1.208 | train_step_timing in s: 2.354 | consumed_samples: 17024 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 532/999 | lr: 4.884e-05 | global_batch_size: 32 | global_step: 532 | reduced_train_loss: 1.121 | train_step_timing in s: 2.389 | consumed_samples: 17056 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 533/999 | lr: 4.868e-05 | global_batch_size: 32 | global_step: 533 | reduced_train_loss: 0.9866 | train_step_timing in s: 2.375 | consumed_samples: 17088 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 534/999 | lr: 4.851e-05 | global_batch_size: 32 | global_step: 534 | reduced_train_loss: 1.096 | train_step_timing in s: 2.32 | consumed_samples: 17120 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 535/999 | lr: 4.835e-05 | global_batch_size: 32 | global_step: 535 | reduced_train_loss: 0.9857 | train_step_timing in s: 2.428 | consumed_samples: 17152 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 536/999 | lr: 4.818e-05 | global_batch_size: 32 | global_step: 536 | reduced_train_loss: 1.178 | train_step_timing in s: 2.449 | consumed_samples: 17184 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 537/999 | lr: 4.802e-05 | global_batch_size: 32 | global_step: 537 | reduced_train_loss: 1.371 | train_step_timing in s: 2.361 | consumed_samples: 17216 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 538/999 | lr: 4.785e-05 | global_batch_size: 32 | global_step: 538 | reduced_train_loss: 1.094 | train_step_timing in s: 2.453 | consumed_samples: 17248 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 539/999 | lr: 4.769e-05 | global_batch_size: 32 | global_step: 539 | reduced_train_loss: 1.182 | train_step_timing in s: 2.39 | consumed_samples: 17280 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:36:45 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 540 : Start time: 1764081405.715s : Save duration: 0.169s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:36:46 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=539-consumed_samples=17280.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 540/999 | lr: 4.752e-05 | global_batch_size: 32 | global_step: 540 | reduced_train_loss: 1.039 | train_step_timing in s: 2.781 | consumed_samples: 17312 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:36:49 pl:222] Successfully saved checkpoint from iteration     540 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=539-consumed_samples=17280.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:36:49 model_checkpoint:657] Async checkpoint save for step 540 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=539-consumed_samples=17280.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:36:49 dist_ckpt_io:160] Async finalization time took 0.111 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 541/999 | lr: 4.736e-05 | global_batch_size: 32 | global_step: 541 | reduced_train_loss: 1.263 | train_step_timing in s: 2.308 | consumed_samples: 17344 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 542/999 | lr: 4.719e-05 | global_batch_size: 32 | global_step: 542 | reduced_train_loss: 0.9583 | train_step_timing in s: 2.378 | consumed_samples: 17376 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 543/999 | lr: 4.703e-05 | global_batch_size: 32 | global_step: 543 | reduced_train_loss: 0.8335 | train_step_timing in s: 2.386 | consumed_samples: 17408 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 544/999 | lr: 4.686e-05 | global_batch_size: 32 | global_step: 544 | reduced_train_loss: 0.9611 | train_step_timing in s: 2.36 | consumed_samples: 17440 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 545/999 | lr: 4.67e-05 | global_batch_size: 32 | global_step: 545 | reduced_train_loss: 1.149 | train_step_timing in s: 2.346 | consumed_samples: 17472 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 546/999 | lr: 4.653e-05 | global_batch_size: 32 | global_step: 546 | reduced_train_loss: 0.9963 | train_step_timing in s: 2.401 | consumed_samples: 17504 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 547/999 | lr: 4.637e-05 | global_batch_size: 32 | global_step: 547 | reduced_train_loss: 1.223 | train_step_timing in s: 2.402 | consumed_samples: 17536 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 548/999 | lr: 4.62e-05 | global_batch_size: 32 | global_step: 548 | reduced_train_loss: 1.225 | train_step_timing in s: 2.361 | consumed_samples: 17568 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 549/999 | lr: 4.604e-05 | global_batch_size: 32 | global_step: 549 | reduced_train_loss: 1.078 | train_step_timing in s: 2.396 | consumed_samples: 17600 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 549: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:37:11 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 549 : Start time: 1764081430.676s : Save duration: 0.606s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:37:11 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=549-consumed_samples=17600.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:37:11 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 550/999 | lr: 4.587e-05 | global_batch_size: 32 | global_step: 550 | reduced_train_loss: 1.079 | train_step_timing in s: 2.865 | consumed_samples: 17632 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:37:14 pl:222] Successfully saved checkpoint from iteration     549 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=549-consumed_samples=17600.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:37:14 model_checkpoint:657] Async checkpoint save for step 550 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=549-consumed_samples=17600.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:37:14 dist_ckpt_io:160] Async finalization time took 0.116 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 551/999 | lr: 4.571e-05 | global_batch_size: 32 | global_step: 551 | reduced_train_loss: 1.086 | train_step_timing in s: 2.42 | consumed_samples: 17664 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 552/999 | lr: 4.554e-05 | global_batch_size: 32 | global_step: 552 | reduced_train_loss: 1.291 | train_step_timing in s: 2.351 | consumed_samples: 17696 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 553/999 | lr: 4.538e-05 | global_batch_size: 32 | global_step: 553 | reduced_train_loss: 1.185 | train_step_timing in s: 2.383 | consumed_samples: 17728 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 554/999 | lr: 4.521e-05 | global_batch_size: 32 | global_step: 554 | reduced_train_loss: 1.129 | train_step_timing in s: 2.468 | consumed_samples: 17760 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 555/999 | lr: 4.505e-05 | global_batch_size: 32 | global_step: 555 | reduced_train_loss: 1.06 | train_step_timing in s: 2.427 | consumed_samples: 17792 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 556/999 | lr: 4.488e-05 | global_batch_size: 32 | global_step: 556 | reduced_train_loss: 1.076 | train_step_timing in s: 2.414 | consumed_samples: 17824 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 557/999 | lr: 4.472e-05 | global_batch_size: 32 | global_step: 557 | reduced_train_loss: 1.148 | train_step_timing in s: 2.443 | consumed_samples: 17856 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 558/999 | lr: 4.455e-05 | global_batch_size: 32 | global_step: 558 | reduced_train_loss: 1.088 | train_step_timing in s: 2.431 | consumed_samples: 17888 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 559/999 | lr: 4.439e-05 | global_batch_size: 32 | global_step: 559 | reduced_train_loss: 1.105 | train_step_timing in s: 2.397 | consumed_samples: 17920 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 560/999 | lr: 4.423e-05 | global_batch_size: 32 | global_step: 560 | reduced_train_loss: 1.032 | train_step_timing in s: 2.414 | consumed_samples: 17952 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 561/999 | lr: 4.406e-05 | global_batch_size: 32 | global_step: 561 | reduced_train_loss: 1.113 | train_step_timing in s: 2.371 | consumed_samples: 17984 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 562/999 | lr: 4.39e-05 | global_batch_size: 32 | global_step: 562 | reduced_train_loss: 1.128 | train_step_timing in s: 2.428 | consumed_samples: 18016 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 563/999 | lr: 4.373e-05 | global_batch_size: 32 | global_step: 563 | reduced_train_loss: 1.128 | train_step_timing in s: 2.398 | consumed_samples: 18048 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 564/999 | lr: 4.357e-05 | global_batch_size: 32 | global_step: 564 | reduced_train_loss: 1.329 | train_step_timing in s: 2.404 | consumed_samples: 18080 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 565/999 | lr: 4.341e-05 | global_batch_size: 32 | global_step: 565 | reduced_train_loss: 1.188 | train_step_timing in s: 2.536 | consumed_samples: 18112 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 566/999 | lr: 4.324e-05 | global_batch_size: 32 | global_step: 566 | reduced_train_loss: 1.28 | train_step_timing in s: 2.447 | consumed_samples: 18144 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 567/999 | lr: 4.308e-05 | global_batch_size: 32 | global_step: 567 | reduced_train_loss: 1.073 | train_step_timing in s: 2.543 | consumed_samples: 18176 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 568/999 | lr: 4.291e-05 | global_batch_size: 32 | global_step: 568 | reduced_train_loss: 1.166 | train_step_timing in s: 2.493 | consumed_samples: 18208 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 569/999 | lr: 4.275e-05 | global_batch_size: 32 | global_step: 569 | reduced_train_loss: 0.9721 | train_step_timing in s: 2.497 | consumed_samples: 18240 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0[default0]:
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:38:59 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 570 : Start time: 1764081539.358s : Save duration: 0.182s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:38:59 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=569-consumed_samples=18240.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 570/999 | lr: 4.259e-05 | global_batch_size: 32 | global_step: 570 | reduced_train_loss: 1.203 | train_step_timing in s: 2.842 | consumed_samples: 18272 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:39:02 pl:222] Successfully saved checkpoint from iteration     570 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=569-consumed_samples=18240.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:39:02 model_checkpoint:657] Async checkpoint save for step 570 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=569-consumed_samples=18240.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:39:02 dist_ckpt_io:160] Async finalization time took 0.111 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 571/999 | lr: 4.242e-05 | global_batch_size: 32 | global_step: 571 | reduced_train_loss: 1.206 | train_step_timing in s: 2.414 | consumed_samples: 18304 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 572/999 | lr: 4.226e-05 | global_batch_size: 32 | global_step: 572 | reduced_train_loss: 1.243 | train_step_timing in s: 2.423 | consumed_samples: 18336 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 573/999 | lr: 4.21e-05 | global_batch_size: 32 | global_step: 573 | reduced_train_loss: 1.266 | train_step_timing in s: 2.444 | consumed_samples: 18368 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 574/999 | lr: 4.193e-05 | global_batch_size: 32 | global_step: 574 | reduced_train_loss: 1.366 | train_step_timing in s: 2.548 | consumed_samples: 18400 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 575/999 | lr: 4.177e-05 | global_batch_size: 32 | global_step: 575 | reduced_train_loss: 1.24 | train_step_timing in s: 2.572 | consumed_samples: 18432 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 576/999 | lr: 4.161e-05 | global_batch_size: 32 | global_step: 576 | reduced_train_loss: 1.115 | train_step_timing in s: 2.463 | consumed_samples: 18464 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 577/999 | lr: 4.144e-05 | global_batch_size: 32 | global_step: 577 | reduced_train_loss: 1.082 | train_step_timing in s: 2.521 | consumed_samples: 18496 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 578/999 | lr: 4.128e-05 | global_batch_size: 32 | global_step: 578 | reduced_train_loss: 1.084 | train_step_timing in s: 2.373 | consumed_samples: 18528 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 579/999 | lr: 4.112e-05 | global_batch_size: 32 | global_step: 579 | reduced_train_loss: 1.153 | train_step_timing in s: 2.45 | consumed_samples: 18560 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 580/999 | lr: 4.096e-05 | global_batch_size: 32 | global_step: 580 | reduced_train_loss: 1.265 | train_step_timing in s: 2.467 | consumed_samples: 18592 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 581/999 | lr: 4.079e-05 | global_batch_size: 32 | global_step: 581 | reduced_train_loss: 1.128 | train_step_timing in s: 2.429 | consumed_samples: 18624 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 582/999 | lr: 4.063e-05 | global_batch_size: 32 | global_step: 582 | reduced_train_loss: 1.101 | train_step_timing in s: 2.43 | consumed_samples: 18656 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 583/999 | lr: 4.047e-05 | global_batch_size: 32 | global_step: 583 | reduced_train_loss: 0.967 | train_step_timing in s: 2.374 | consumed_samples: 18688 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 584/999 | lr: 4.031e-05 | global_batch_size: 32 | global_step: 584 | reduced_train_loss: 1.089 | train_step_timing in s: 2.451 | consumed_samples: 18720 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 585/999 | lr: 4.014e-05 | global_batch_size: 32 | global_step: 585 | reduced_train_loss: 1.427 | train_step_timing in s: 2.392 | consumed_samples: 18752 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 586/999 | lr: 3.998e-05 | global_batch_size: 32 | global_step: 586 | reduced_train_loss: 1.154 | train_step_timing in s: 2.337 | consumed_samples: 18784 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 587/999 | lr: 3.982e-05 | global_batch_size: 32 | global_step: 587 | reduced_train_loss: 1.187 | train_step_timing in s: 2.386 | consumed_samples: 18816 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 588/999 | lr: 3.966e-05 | global_batch_size: 32 | global_step: 588 | reduced_train_loss: 1.021 | train_step_timing in s: 2.383 | consumed_samples: 18848 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 589/999 | lr: 3.95e-05 | global_batch_size: 32 | global_step: 589 | reduced_train_loss: 1.244 | train_step_timing in s: 2.375 | consumed_samples: 18880 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 590/999 | lr: 3.934e-05 | global_batch_size: 32 | global_step: 590 | reduced_train_loss: 1.12 | train_step_timing in s: 2.368 | consumed_samples: 18912 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 591/999 | lr: 3.917e-05 | global_batch_size: 32 | global_step: 591 | reduced_train_loss: 1.388 | train_step_timing in s: 2.425 | consumed_samples: 18944 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 592/999 | lr: 3.901e-05 | global_batch_size: 32 | global_step: 592 | reduced_train_loss: 1.085 | train_step_timing in s: 2.444 | consumed_samples: 18976 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 593/999 | lr: 3.885e-05 | global_batch_size: 32 | global_step: 593 | reduced_train_loss: 1.26 | train_step_timing in s: 2.446 | consumed_samples: 19008 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 594/999 | lr: 3.869e-05 | global_batch_size: 32 | global_step: 594 | reduced_train_loss: 1.184 | train_step_timing in s: 2.396 | consumed_samples: 19040 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 595/999 | lr: 3.853e-05 | global_batch_size: 32 | global_step: 595 | reduced_train_loss: 0.9901 | train_step_timing in s: 2.443 | consumed_samples: 19072 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 596/999 | lr: 3.837e-05 | global_batch_size: 32 | global_step: 596 | reduced_train_loss: 1.001 | train_step_timing in s: 2.44 | consumed_samples: 19104 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 597/999 | lr: 3.821e-05 | global_batch_size: 32 | global_step: 597 | reduced_train_loss: 0.9613 | train_step_timing in s: 2.527 | consumed_samples: 19136 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 598/999 | lr: 3.805e-05 | global_batch_size: 32 | global_step: 598 | reduced_train_loss: 1.113 | train_step_timing in s: 2.44 | consumed_samples: 19168 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 599/999 | lr: 3.789e-05 | global_batch_size: 32 | global_step: 599 | reduced_train_loss: 1.096 | train_step_timing in s: 2.385 | consumed_samples: 19200 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 599: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:40:14 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 599 : Start time: 1764081614.014s : Save duration: 0.180s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:40:14 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=599-consumed_samples=19200.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:40:14 dist_ckpt_io:160] Async finalization time took 0.002 s
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:Training epoch 0, iteration 600/999 | lr: 3.773e-05 | global_batch_size: 32 | global_step: 600 | reduced_train_loss: 1.069 | train_step_timing in s: 2.59 | consumed_samples: 19232 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:15 pl:222] Successfully saved checkpoint from iteration     599 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=599-consumed_samples=19200.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:15 model_checkpoint:657] Async checkpoint save for step 600 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=599-consumed_samples=19200.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:15 dist_ckpt_io:160] Async finalization time took 0.120 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 601/999 | lr: 3.757e-05 | global_batch_size: 32 | global_step: 601 | reduced_train_loss: 0.9534 | train_step_timing in s: 2.515 | consumed_samples: 19264 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 602/999 | lr: 3.741e-05 | global_batch_size: 32 | global_step: 602 | reduced_train_loss: 1.122 | train_step_timing in s: 2.516 | consumed_samples: 19296 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 603/999 | lr: 3.725e-05 | global_batch_size: 32 | global_step: 603 | reduced_train_loss: 1.118 | train_step_timing in s: 2.408 | consumed_samples: 19328 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 604/999 | lr: 3.709e-05 | global_batch_size: 32 | global_step: 604 | reduced_train_loss: 1.201 | train_step_timing in s: 2.49 | consumed_samples: 19360 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 605/999 | lr: 3.693e-05 | global_batch_size: 32 | global_step: 605 | reduced_train_loss: 1.339 | train_step_timing in s: 2.384 | consumed_samples: 19392 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 606/999 | lr: 3.677e-05 | global_batch_size: 32 | global_step: 606 | reduced_train_loss: 1.112 | train_step_timing in s: 2.475 | consumed_samples: 19424 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 607/999 | lr: 3.661e-05 | global_batch_size: 32 | global_step: 607 | reduced_train_loss: 1.165 | train_step_timing in s: 2.48 | consumed_samples: 19456 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 608/999 | lr: 3.645e-05 | global_batch_size: 32 | global_step: 608 | reduced_train_loss: 1.083 | train_step_timing in s: 2.455 | consumed_samples: 19488 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 609/999 | lr: 3.629e-05 | global_batch_size: 32 | global_step: 609 | reduced_train_loss: 1.112 | train_step_timing in s: 2.638 | consumed_samples: 19520 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 610/999 | lr: 3.613e-05 | global_batch_size: 32 | global_step: 610 | reduced_train_loss: 1.21 | train_step_timing in s: 2.516 | consumed_samples: 19552 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 611/999 | lr: 3.597e-05 | global_batch_size: 32 | global_step: 611 | reduced_train_loss: 0.9539 | train_step_timing in s: 2.527 | consumed_samples: 19584 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 612/999 | lr: 3.581e-05 | global_batch_size: 32 | global_step: 612 | reduced_train_loss: 1.087 | train_step_timing in s: 2.516 | consumed_samples: 19616 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 613/999 | lr: 3.565e-05 | global_batch_size: 32 | global_step: 613 | reduced_train_loss: 1.277 | train_step_timing in s: 2.468 | consumed_samples: 19648 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 614/999 | lr: 3.55e-05 | global_batch_size: 32 | global_step: 614 | reduced_train_loss: 1.053 | train_step_timing in s: 2.56 | consumed_samples: 19680 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 615/999 | lr: 3.534e-05 | global_batch_size: 32 | global_step: 615 | reduced_train_loss: 1.109 | train_step_timing in s: 2.443 | consumed_samples: 19712 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 616/999 | lr: 3.518e-05 | global_batch_size: 32 | global_step: 616 | reduced_train_loss: 1.14 | train_step_timing in s: 2.407 | consumed_samples: 19744 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 617/999 | lr: 3.502e-05 | global_batch_size: 32 | global_step: 617 | reduced_train_loss: 1.347 | train_step_timing in s: 2.404 | consumed_samples: 19776 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 618/999 | lr: 3.486e-05 | global_batch_size: 32 | global_step: 618 | reduced_train_loss: 0.8645 | train_step_timing in s: 2.469 | consumed_samples: 19808 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 619/999 | lr: 3.471e-05 | global_batch_size: 32 | global_step: 619 | reduced_train_loss: 1.212 | train_step_timing in s: 2.439 | consumed_samples: 19840 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 620/999 | lr: 3.455e-05 | global_batch_size: 32 | global_step: 620 | reduced_train_loss: 1.198 | train_step_timing in s: 2.45 | consumed_samples: 19872 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 621/999 | lr: 3.439e-05 | global_batch_size: 32 | global_step: 621 | reduced_train_loss: 1.078 | train_step_timing in s: 2.412 | consumed_samples: 19904 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 622/999 | lr: 3.423e-05 | global_batch_size: 32 | global_step: 622 | reduced_train_loss: 1.113 | train_step_timing in s: 2.421 | consumed_samples: 19936 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 623/999 | lr: 3.408e-05 | global_batch_size: 32 | global_step: 623 | reduced_train_loss: 1.122 | train_step_timing in s: 2.458 | consumed_samples: 19968 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 624/999 | lr: 3.392e-05 | global_batch_size: 32 | global_step: 624 | reduced_train_loss: 1.068 | train_step_timing in s: 2.4 | consumed_samples: 20000 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 625/999 | lr: 3.377e-05 | global_batch_size: 32 | global_step: 625 | reduced_train_loss: 0.9365 | train_step_timing in s: 2.409 | consumed_samples: 20032 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 626/999 | lr: 3.361e-05 | global_batch_size: 32 | global_step: 626 | reduced_train_loss: 1.15 | train_step_timing in s: 2.372 | consumed_samples: 20064 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 627/999 | lr: 3.345e-05 | global_batch_size: 32 | global_step: 627 | reduced_train_loss: 1.034 | train_step_timing in s: 2.382 | consumed_samples: 20096 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 628/999 | lr: 3.33e-05 | global_batch_size: 32 | global_step: 628 | reduced_train_loss: 1.189 | train_step_timing in s: 2.369 | consumed_samples: 20128 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 629/999 | lr: 3.314e-05 | global_batch_size: 32 | global_step: 629 | reduced_train_loss: 1.254 | train_step_timing in s: 2.35 | consumed_samples: 20160 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0s_act_ckpt/0 [default0]:
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:24 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 630 : Start time: 1764081804.360s : Save duration: 0.631s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:25 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=629-consumed_samples=20160.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 630/999 | lr: 3.299e-05 | global_batch_size: 32 | global_step: 630 | reduced_train_loss: 1.229 | train_step_timing in s: 2.788 | consumed_samples: 20192 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:28 pl:222] Successfully saved checkpoint from iteration     630 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=629-consumed_samples=20160.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:28 model_checkpoint:657] Async checkpoint save for step 630 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=629-consumed_samples=20160.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:28 dist_ckpt_io:160] Async finalization time took 0.115 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 631/999 | lr: 3.283e-05 | global_batch_size: 32 | global_step: 631 | reduced_train_loss: 0.9972 | train_step_timing in s: 2.4 | consumed_samples: 20224 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 632/999 | lr: 3.267e-05 | global_batch_size: 32 | global_step: 632 | reduced_train_loss: 1.315 | train_step_timing in s: 2.414 | consumed_samples: 20256 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 633/999 | lr: 3.252e-05 | global_batch_size: 32 | global_step: 633 | reduced_train_loss: 1.267 | train_step_timing in s: 2.374 | consumed_samples: 20288 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 634/999 | lr: 3.236e-05 | global_batch_size: 32 | global_step: 634 | reduced_train_loss: 1.041 | train_step_timing in s: 2.372 | consumed_samples: 20320 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 635/999 | lr: 3.221e-05 | global_batch_size: 32 | global_step: 635 | reduced_train_loss: 1.189 | train_step_timing in s: 2.385 | consumed_samples: 20352 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 636/999 | lr: 3.206e-05 | global_batch_size: 32 | global_step: 636 | reduced_train_loss: 1.131 | train_step_timing in s: 2.38 | consumed_samples: 20384 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 637/999 | lr: 3.19e-05 | global_batch_size: 32 | global_step: 637 | reduced_train_loss: 0.9643 | train_step_timing in s: 2.423 | consumed_samples: 20416 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 638/999 | lr: 3.175e-05 | global_batch_size: 32 | global_step: 638 | reduced_train_loss: 1.31 | train_step_timing in s: 2.39 | consumed_samples: 20448 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 639/999 | lr: 3.159e-05 | global_batch_size: 32 | global_step: 639 | reduced_train_loss: 0.9123 | train_step_timing in s: 2.455 | consumed_samples: 20480 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 640/999 | lr: 3.144e-05 | global_batch_size: 32 | global_step: 640 | reduced_train_loss: 1.166 | train_step_timing in s: 2.403 | consumed_samples: 20512 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 641/999 | lr: 3.129e-05 | global_batch_size: 32 | global_step: 641 | reduced_train_loss: 1.119 | train_step_timing in s: 2.353 | consumed_samples: 20544 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 642/999 | lr: 3.113e-05 | global_batch_size: 32 | global_step: 642 | reduced_train_loss: 1.257 | train_step_timing in s: 2.376 | consumed_samples: 20576 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 643/999 | lr: 3.098e-05 | global_batch_size: 32 | global_step: 643 | reduced_train_loss: 1.12 | train_step_timing in s: 2.347 | consumed_samples: 20608 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 644/999 | lr: 3.083e-05 | global_batch_size: 32 | global_step: 644 | reduced_train_loss: 1.033 | train_step_timing in s: 2.465 | consumed_samples: 20640 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 645/999 | lr: 3.068e-05 | global_batch_size: 32 | global_step: 645 | reduced_train_loss: 1.158 | train_step_timing in s: 2.397 | consumed_samples: 20672 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 646/999 | lr: 3.052e-05 | global_batch_size: 32 | global_step: 646 | reduced_train_loss: 1.188 | train_step_timing in s: 2.371 | consumed_samples: 20704 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 647/999 | lr: 3.037e-05 | global_batch_size: 32 | global_step: 647 | reduced_train_loss: 1.222 | train_step_timing in s: 2.408 | consumed_samples: 20736 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 648/999 | lr: 3.022e-05 | global_batch_size: 32 | global_step: 648 | reduced_train_loss: 1.241 | train_step_timing in s: 2.455 | consumed_samples: 20768 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 649/999 | lr: 3.007e-05 | global_batch_size: 32 | global_step: 649 | reduced_train_loss: 1.09 | train_step_timing in s: 2.393 | consumed_samples: 20800 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 649: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:14 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 649 : Start time: 1764081854.140s : Save duration: 0.174s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:14 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=649-consumed_samples=20800.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:14 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 650/999 | lr: 2.992e-05 | global_batch_size: 32 | global_step: 650 | reduced_train_loss: 1.168 | train_step_timing in s: 2.678 | consumed_samples: 20832 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:17 pl:222] Successfully saved checkpoint from iteration     649 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=649-consumed_samples=20800.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:17 model_checkpoint:657] Async checkpoint save for step 650 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=649-consumed_samples=20800.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:17 dist_ckpt_io:160] Async finalization time took 0.121 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 651/999 | lr: 2.976e-05 | global_batch_size: 32 | global_step: 651 | reduced_train_loss: 1.24 | train_step_timing in s: 2.478 | consumed_samples: 20864 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 652/999 | lr: 2.961e-05 | global_batch_size: 32 | global_step: 652 | reduced_train_loss: 1.252 | train_step_timing in s: 2.479 | consumed_samples: 20896 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 653/999 | lr: 2.946e-05 | global_batch_size: 32 | global_step: 653 | reduced_train_loss: 1.075 | train_step_timing in s: 2.426 | consumed_samples: 20928 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 654/999 | lr: 2.931e-05 | global_batch_size: 32 | global_step: 654 | reduced_train_loss: 1.141 | train_step_timing in s: 2.416 | consumed_samples: 20960 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 655/999 | lr: 2.916e-05 | global_batch_size: 32 | global_step: 655 | reduced_train_loss: 0.991 | train_step_timing in s: 2.466 | consumed_samples: 20992 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 656/999 | lr: 2.901e-05 | global_batch_size: 32 | global_step: 656 | reduced_train_loss: 1.286 | train_step_timing in s: 2.423 | consumed_samples: 21024 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 657/999 | lr: 2.886e-05 | global_batch_size: 32 | global_step: 657 | reduced_train_loss: 0.9881 | train_step_timing in s: 2.365 | consumed_samples: 21056 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 658/999 | lr: 2.871e-05 | global_batch_size: 32 | global_step: 658 | reduced_train_loss: 1.127 | train_step_timing in s: 2.385 | consumed_samples: 21088 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 659/999 | lr: 2.856e-05 | global_batch_size: 32 | global_step: 659 | reduced_train_loss: 1.109 | train_step_timing in s: 2.465 | consumed_samples: 21120 | val_loss: 1.165[default0]:
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:36 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 660 : Start time: 1764081936.798s : Save duration: 0.171s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:37 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=659-consumed_samples=21120.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 660/999 | lr: 2.841e-05 | global_batch_size: 32 | global_step: 660 | reduced_train_loss: 1.053 | train_step_timing in s: 2.559 | consumed_samples: 21152 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:39 pl:222] Successfully saved checkpoint from iteration     660 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=659-consumed_samples=21120.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:39 model_checkpoint:657] Async checkpoint save for step 660 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=659-consumed_samples=21120.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:40 dist_ckpt_io:160] Async finalization time took 0.115 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 661/999 | lr: 2.826e-05 | global_batch_size: 32 | global_step: 661 | reduced_train_loss: 1.308 | train_step_timing in s: 2.396 | consumed_samples: 21184 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 662/999 | lr: 2.811e-05 | global_batch_size: 32 | global_step: 662 | reduced_train_loss: 1.094 | train_step_timing in s: 2.444 | consumed_samples: 21216 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 663/999 | lr: 2.797e-05 | global_batch_size: 32 | global_step: 663 | reduced_train_loss: 1.088 | train_step_timing in s: 2.394 | consumed_samples: 21248 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 664/999 | lr: 2.782e-05 | global_batch_size: 32 | global_step: 664 | reduced_train_loss: 1.274 | train_step_timing in s: 2.401 | consumed_samples: 21280 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 665/999 | lr: 2.767e-05 | global_batch_size: 32 | global_step: 665 | reduced_train_loss: 1.209 | train_step_timing in s: 2.414 | consumed_samples: 21312 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 666/999 | lr: 2.752e-05 | global_batch_size: 32 | global_step: 666 | reduced_train_loss: 1.122 | train_step_timing in s: 2.404 | consumed_samples: 21344 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 667/999 | lr: 2.737e-05 | global_batch_size: 32 | global_step: 667 | reduced_train_loss: 1.117 | train_step_timing in s: 2.4 | consumed_samples: 21376 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 668/999 | lr: 2.723e-05 | global_batch_size: 32 | global_step: 668 | reduced_train_loss: 1.224 | train_step_timing in s: 2.397 | consumed_samples: 21408 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 669/999 | lr: 2.708e-05 | global_batch_size: 32 | global_step: 669 | reduced_train_loss: 1.062 | train_step_timing in s: 2.369 | consumed_samples: 21440 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 670/999 | lr: 2.693e-05 | global_batch_size: 32 | global_step: 670 | reduced_train_loss: 1.165 | train_step_timing in s: 2.381 | consumed_samples: 21472 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 671/999 | lr: 2.679e-05 | global_batch_size: 32 | global_step: 671 | reduced_train_loss: 1.189 | train_step_timing in s: 2.398 | consumed_samples: 21504 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 672/999 | lr: 2.664e-05 | global_batch_size: 32 | global_step: 672 | reduced_train_loss: 1.25 | train_step_timing in s: 2.408 | consumed_samples: 21536 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 673/999 | lr: 2.649e-05 | global_batch_size: 32 | global_step: 673 | reduced_train_loss: 1.119 | train_step_timing in s: 2.423 | consumed_samples: 21568 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 674/999 | lr: 2.635e-05 | global_batch_size: 32 | global_step: 674 | reduced_train_loss: 1.176 | train_step_timing in s: 2.37 | consumed_samples: 21600 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 675/999 | lr: 2.62e-05 | global_batch_size: 32 | global_step: 675 | reduced_train_loss: 1.093 | train_step_timing in s: 2.366 | consumed_samples: 21632 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 676/999 | lr: 2.606e-05 | global_batch_size: 32 | global_step: 676 | reduced_train_loss: 1.187 | train_step_timing in s: 2.41 | consumed_samples: 21664 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 677/999 | lr: 2.591e-05 | global_batch_size: 32 | global_step: 677 | reduced_train_loss: 1.094 | train_step_timing in s: 2.361 | consumed_samples: 21696 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 678/999 | lr: 2.577e-05 | global_batch_size: 32 | global_step: 678 | reduced_train_loss: 1.096 | train_step_timing in s: 2.396 | consumed_samples: 21728 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 679/999 | lr: 2.562e-05 | global_batch_size: 32 | global_step: 679 | reduced_train_loss: 1.011 | train_step_timing in s: 2.361 | consumed_samples: 21760 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 680/999 | lr: 2.548e-05 | global_batch_size: 32 | global_step: 680 | reduced_train_loss: 1.039 | train_step_timing in s: 2.396 | consumed_samples: 21792 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 681/999 | lr: 2.533e-05 | global_batch_size: 32 | global_step: 681 | reduced_train_loss: 1.281 | train_step_timing in s: 2.329 | consumed_samples: 21824 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 682/999 | lr: 2.519e-05 | global_batch_size: 32 | global_step: 682 | reduced_train_loss: 1.363 | train_step_timing in s: 2.33 | consumed_samples: 21856 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 683/999 | lr: 2.505e-05 | global_batch_size: 32 | global_step: 683 | reduced_train_loss: 1.194 | train_step_timing in s: 2.443 | consumed_samples: 21888 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 684/999 | lr: 2.49e-05 | global_batch_size: 32 | global_step: 684 | reduced_train_loss: 1.071 | train_step_timing in s: 2.373 | consumed_samples: 21920 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 685/999 | lr: 2.476e-05 | global_batch_size: 32 | global_step: 685 | reduced_train_loss: 1.04 | train_step_timing in s: 2.366 | consumed_samples: 21952 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 686/999 | lr: 2.462e-05 | global_batch_size: 32 | global_step: 686 | reduced_train_loss: 0.9715 | train_step_timing in s: 2.425 | consumed_samples: 21984 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 687/999 | lr: 2.448e-05 | global_batch_size: 32 | global_step: 687 | reduced_train_loss: 1.211 | train_step_timing in s: 2.397 | consumed_samples: 22016 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 688/999 | lr: 2.433e-05 | global_batch_size: 32 | global_step: 688 | reduced_train_loss: 0.9193 | train_step_timing in s: 2.372 | consumed_samples: 22048 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 689/999 | lr: 2.419e-05 | global_batch_size: 32 | global_step: 689 | reduced_train_loss: 1.187 | train_step_timing in s: 2.396 | consumed_samples: 22080 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:47 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 690 : Start time: 1764082066.863s : Save duration: 0.175s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:47 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=689-consumed_samples=22080.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 690/999 | lr: 2.405e-05 | global_batch_size: 32 | global_step: 690 | reduced_train_loss: 1.044 | train_step_timing in s: 3.205 | consumed_samples: 22112 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:51 pl:222] Successfully saved checkpoint from iteration     690 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=689-consumed_samples=22080.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:51 model_checkpoint:657] Async checkpoint save for step 690 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=689-consumed_samples=22080.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:51 dist_ckpt_io:160] Async finalization time took 0.581 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 691/999 | lr: 2.391e-05 | global_batch_size: 32 | global_step: 691 | reduced_train_loss: 1.222 | train_step_timing in s: 2.403 | consumed_samples: 22144 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 692/999 | lr: 2.377e-05 | global_batch_size: 32 | global_step: 692 | reduced_train_loss: 1.072 | train_step_timing in s: 2.35 | consumed_samples: 22176 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 693/999 | lr: 2.363e-05 | global_batch_size: 32 | global_step: 693 | reduced_train_loss: 1.218 | train_step_timing in s: 2.397 | consumed_samples: 22208 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 694/999 | lr: 2.349e-05 | global_batch_size: 32 | global_step: 694 | reduced_train_loss: 1.137 | train_step_timing in s: 2.385 | consumed_samples: 22240 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 695/999 | lr: 2.335e-05 | global_batch_size: 32 | global_step: 695 | reduced_train_loss: 1.091 | train_step_timing in s: 2.421 | consumed_samples: 22272 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 696/999 | lr: 2.321e-05 | global_batch_size: 32 | global_step: 696 | reduced_train_loss: 1.133 | train_step_timing in s: 2.427 | consumed_samples: 22304 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 697/999 | lr: 2.307e-05 | global_batch_size: 32 | global_step: 697 | reduced_train_loss: 1.132 | train_step_timing in s: 2.457 | consumed_samples: 22336 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 698/999 | lr: 2.293e-05 | global_batch_size: 32 | global_step: 698 | reduced_train_loss: 1.121 | train_step_timing in s: 2.479 | consumed_samples: 22368 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 699/999 | lr: 2.279e-05 | global_batch_size: 32 | global_step: 699 | reduced_train_loss: 1.009 | train_step_timing in s: 2.38 | consumed_samples: 22400 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 699: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:13 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 699 : Start time: 1764082093.154s : Save duration: 0.170s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:13 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=699-consumed_samples=22400.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:13 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 700/999 | lr: 2.265e-05 | global_batch_size: 32 | global_step: 700 | reduced_train_loss: 1.089 | train_step_timing in s: 2.964 | consumed_samples: 22432 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:16 pl:222] Successfully saved checkpoint from iteration     699 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=699-consumed_samples=22400.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:16 model_checkpoint:657] Async checkpoint save for step 700 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=699-consumed_samples=22400.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:16 dist_ckpt_io:160] Async finalization time took 0.126 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 701/999 | lr: 2.251e-05 | global_batch_size: 32 | global_step: 701 | reduced_train_loss: 0.972 | train_step_timing in s: 2.441 | consumed_samples: 22464 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 702/999 | lr: 2.238e-05 | global_batch_size: 32 | global_step: 702 | reduced_train_loss: 1.308 | train_step_timing in s: 2.458 | consumed_samples: 22496 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 703/999 | lr: 2.224e-05 | global_batch_size: 32 | global_step: 703 | reduced_train_loss: 1.104 | train_step_timing in s: 2.394 | consumed_samples: 22528 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 704/999 | lr: 2.21e-05 | global_batch_size: 32 | global_step: 704 | reduced_train_loss: 1.344 | train_step_timing in s: 2.399 | consumed_samples: 22560 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 705/999 | lr: 2.196e-05 | global_batch_size: 32 | global_step: 705 | reduced_train_loss: 1.18 | train_step_timing in s: 2.377 | consumed_samples: 22592 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 706/999 | lr: 2.183e-05 | global_batch_size: 32 | global_step: 706 | reduced_train_loss: 1.006 | train_step_timing in s: 2.35 | consumed_samples: 22624 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 707/999 | lr: 2.169e-05 | global_batch_size: 32 | global_step: 707 | reduced_train_loss: 1.238 | train_step_timing in s: 2.392 | consumed_samples: 22656 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 708/999 | lr: 2.155e-05 | global_batch_size: 32 | global_step: 708 | reduced_train_loss: 1.062 | train_step_timing in s: 2.474 | consumed_samples: 22688 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 709/999 | lr: 2.142e-05 | global_batch_size: 32 | global_step: 709 | reduced_train_loss: 1.111 | train_step_timing in s: 2.451 | consumed_samples: 22720 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 710/999 | lr: 2.128e-05 | global_batch_size: 32 | global_step: 710 | reduced_train_loss: 1.009 | train_step_timing in s: 2.441 | consumed_samples: 22752 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 711/999 | lr: 2.115e-05 | global_batch_size: 32 | global_step: 711 | reduced_train_loss: 1.14 | train_step_timing in s: 2.386 | consumed_samples: 22784 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 712/999 | lr: 2.101e-05 | global_batch_size: 32 | global_step: 712 | reduced_train_loss: 1.027 | train_step_timing in s: 2.428 | consumed_samples: 22816 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 713/999 | lr: 2.088e-05 | global_batch_size: 32 | global_step: 713 | reduced_train_loss: 1.134 | train_step_timing in s: 2.473 | consumed_samples: 22848 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 714/999 | lr: 2.074e-05 | global_batch_size: 32 | global_step: 714 | reduced_train_loss: 1.178 | train_step_timing in s: 2.407 | consumed_samples: 22880 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 715/999 | lr: 2.061e-05 | global_batch_size: 32 | global_step: 715 | reduced_train_loss: 1.026 | train_step_timing in s: 2.374 | consumed_samples: 22912 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 716/999 | lr: 2.048e-05 | global_batch_size: 32 | global_step: 716 | reduced_train_loss: 0.9648 | train_step_timing in s: 2.41 | consumed_samples: 22944 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 717/999 | lr: 2.034e-05 | global_batch_size: 32 | global_step: 717 | reduced_train_loss: 1.365 | train_step_timing in s: 2.335 | consumed_samples: 22976 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 718/999 | lr: 2.021e-05 | global_batch_size: 32 | global_step: 718 | reduced_train_loss: 1.086 | train_step_timing in s: 2.368 | consumed_samples: 23008 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 719/999 | lr: 2.008e-05 | global_batch_size: 32 | global_step: 719 | reduced_train_loss: 0.8955 | train_step_timing in s: 2.4 | consumed_samples: 23040 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:50:00 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 720 : Start time: 1764082199.838s : Save duration: 0.168s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:50:00 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=719-consumed_samples=23040.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 720/999 | lr: 1.995e-05 | global_batch_size: 32 | global_step: 720 | reduced_train_loss: 1.054 | train_step_timing in s: 2.744 | consumed_samples: 23072 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:50:03 pl:222] Successfully saved checkpoint from iteration     720 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=719-consumed_samples=23040.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:50:03 model_checkpoint:657] Async checkpoint save for step 720 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=719-consumed_samples=23040.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:50:03 dist_ckpt_io:160] Async finalization time took 0.124 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 721/999 | lr: 1.981e-05 | global_batch_size: 32 | global_step: 721 | reduced_train_loss: 1.268 | train_step_timing in s: 2.366 | consumed_samples: 23104 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 722/999 | lr: 1.968e-05 | global_batch_size: 32 | global_step: 722 | reduced_train_loss: 1.151 | train_step_timing in s: 2.407 | consumed_samples: 23136 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 723/999 | lr: 1.955e-05 | global_batch_size: 32 | global_step: 723 | reduced_train_loss: 1.18 | train_step_timing in s: 2.415 | consumed_samples: 23168 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 724/999 | lr: 1.942e-05 | global_batch_size: 32 | global_step: 724 | reduced_train_loss: 1.099 | train_step_timing in s: 2.464 | consumed_samples: 23200 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 725/999 | lr: 1.929e-05 | global_batch_size: 32 | global_step: 725 | reduced_train_loss: 1.065 | train_step_timing in s: 2.472 | consumed_samples: 23232 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 726/999 | lr: 1.916e-05 | global_batch_size: 32 | global_step: 726 | reduced_train_loss: 1.126 | train_step_timing in s: 2.423 | consumed_samples: 23264 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 727/999 | lr: 1.903e-05 | global_batch_size: 32 | global_step: 727 | reduced_train_loss: 1.018 | train_step_timing in s: 2.494 | consumed_samples: 23296 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 728/999 | lr: 1.89e-05 | global_batch_size: 32 | global_step: 728 | reduced_train_loss: 1.055 | train_step_timing in s: 2.467 | consumed_samples: 23328 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 729/999 | lr: 1.877e-05 | global_batch_size: 32 | global_step: 729 | reduced_train_loss: 1.092 | train_step_timing in s: 2.389 | consumed_samples: 23360 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 730/999 | lr: 1.864e-05 | global_batch_size: 32 | global_step: 730 | reduced_train_loss: 1.164 | train_step_timing in s: 2.358 | consumed_samples: 23392 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 731/999 | lr: 1.851e-05 | global_batch_size: 32 | global_step: 731 | reduced_train_loss: 1.289 | train_step_timing in s: 2.372 | consumed_samples: 23424 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 732/999 | lr: 1.838e-05 | global_batch_size: 32 | global_step: 732 | reduced_train_loss: 1.03 | train_step_timing in s: 2.426 | consumed_samples: 23456 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 733/999 | lr: 1.826e-05 | global_batch_size: 32 | global_step: 733 | reduced_train_loss: 1.087 | train_step_timing in s: 2.439 | consumed_samples: 23488 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 734/999 | lr: 1.813e-05 | global_batch_size: 32 | global_step: 734 | reduced_train_loss: 1.059 | train_step_timing in s: 2.406 | consumed_samples: 23520 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 735/999 | lr: 1.8e-05 | global_batch_size: 32 | global_step: 735 | reduced_train_loss: 1.195 | train_step_timing in s: 2.409 | consumed_samples: 23552 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 736/999 | lr: 1.787e-05 | global_batch_size: 32 | global_step: 736 | reduced_train_loss: 1.313 | train_step_timing in s: 2.403 | consumed_samples: 23584 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 737/999 | lr: 1.775e-05 | global_batch_size: 32 | global_step: 737 | reduced_train_loss: 1.17 | train_step_timing in s: 2.454 | consumed_samples: 23616 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 738/999 | lr: 1.762e-05 | global_batch_size: 32 | global_step: 738 | reduced_train_loss: 1.264 | train_step_timing in s: 2.423 | consumed_samples: 23648 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 739/999 | lr: 1.75e-05 | global_batch_size: 32 | global_step: 739 | reduced_train_loss: 1.142 | train_step_timing in s: 2.452 | consumed_samples: 23680 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 740/999 | lr: 1.737e-05 | global_batch_size: 32 | global_step: 740 | reduced_train_loss: 1.19 | train_step_timing in s: 2.452 | consumed_samples: 23712 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 741/999 | lr: 1.725e-05 | global_batch_size: 32 | global_step: 741 | reduced_train_loss: 1.135 | train_step_timing in s: 2.388 | consumed_samples: 23744 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 742/999 | lr: 1.712e-05 | global_batch_size: 32 | global_step: 742 | reduced_train_loss: 1.194 | train_step_timing in s: 2.442 | consumed_samples: 23776 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 743/999 | lr: 1.7e-05 | global_batch_size: 32 | global_step: 743 | reduced_train_loss: 1.272 | train_step_timing in s: 2.468 | consumed_samples: 23808 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 744/999 | lr: 1.687e-05 | global_batch_size: 32 | global_step: 744 | reduced_train_loss: 1.132 | train_step_timing in s: 2.396 | consumed_samples: 23840 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 745/999 | lr: 1.675e-05 | global_batch_size: 32 | global_step: 745 | reduced_train_loss: 1.212 | train_step_timing in s: 2.399 | consumed_samples: 23872 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 746/999 | lr: 1.663e-05 | global_batch_size: 32 | global_step: 746 | reduced_train_loss: 1.263 | train_step_timing in s: 2.44 | consumed_samples: 23904 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 747/999 | lr: 1.65e-05 | global_batch_size: 32 | global_step: 747 | reduced_train_loss: 1.26 | train_step_timing in s: 2.371 | consumed_samples: 23936 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 748/999 | lr: 1.638e-05 | global_batch_size: 32 | global_step: 748 | reduced_train_loss: 1.096 | train_step_timing in s: 2.44 | consumed_samples: 23968 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 749/999 | lr: 1.626e-05 | global_batch_size: 32 | global_step: 749 | reduced_train_loss: 1.089 | train_step_timing in s: 2.433 | consumed_samples: 24000 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 749: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:51:14 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 749 : Start time: 1764082274.071s : Save duration: 0.176s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:51:14 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=749-consumed_samples=24000.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:51:14 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:Training epoch 0, iteration 750/999 | lr: 1.614e-05 | global_batch_size: 32 | global_step: 750 | reduced_train_loss: 1.147 | train_step_timing in s: 2.401 | consumed_samples: 24032 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:52:14 pl:222] Successfully saved checkpoint from iteration     749 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=749-consumed_samples=24000.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:52:14 model_checkpoint:657] Async checkpoint save for step 750 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=749-consumed_samples=24000.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:52:14 dist_ckpt_io:160] Async finalization time took 0.126 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 751/999 | lr: 1.601e-05 | global_batch_size: 32 | global_step: 751 | reduced_train_loss: 1.016 | train_step_timing in s: 2.466 | consumed_samples: 24064 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 752/999 | lr: 1.589e-05 | global_batch_size: 32 | global_step: 752 | reduced_train_loss: 1.254 | train_step_timing in s: 2.389 | consumed_samples: 24096 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 753/999 | lr: 1.577e-05 | global_batch_size: 32 | global_step: 753 | reduced_train_loss: 1.235 | train_step_timing in s: 2.425 | consumed_samples: 24128 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 754/999 | lr: 1.565e-05 | global_batch_size: 32 | global_step: 754 | reduced_train_loss: 1.33 | train_step_timing in s: 2.377 | consumed_samples: 24160 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 755/999 | lr: 1.553e-05 | global_batch_size: 32 | global_step: 755 | reduced_train_loss: 1.314 | train_step_timing in s: 2.369 | consumed_samples: 24192 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 756/999 | lr: 1.541e-05 | global_batch_size: 32 | global_step: 756 | reduced_train_loss: 1.132 | train_step_timing in s: 2.465 | consumed_samples: 24224 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 757/999 | lr: 1.529e-05 | global_batch_size: 32 | global_step: 757 | reduced_train_loss: 1.167 | train_step_timing in s: 2.413 | consumed_samples: 24256 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 758/999 | lr: 1.517e-05 | global_batch_size: 32 | global_step: 758 | reduced_train_loss: 1.212 | train_step_timing in s: 2.424 | consumed_samples: 24288 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 759/999 | lr: 1.506e-05 | global_batch_size: 32 | global_step: 759 | reduced_train_loss: 1.199 | train_step_timing in s: 2.489 | consumed_samples: 24320 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 760/999 | lr: 1.494e-05 | global_batch_size: 32 | global_step: 760 | reduced_train_loss: 1.188 | train_step_timing in s: 2.423 | consumed_samples: 24352 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 761/999 | lr: 1.482e-05 | global_batch_size: 32 | global_step: 761 | reduced_train_loss: 1.144 | train_step_timing in s: 2.46 | consumed_samples: 24384 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 762/999 | lr: 1.47e-05 | global_batch_size: 32 | global_step: 762 | reduced_train_loss: 0.929 | train_step_timing in s: 2.497 | consumed_samples: 24416 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 763/999 | lr: 1.459e-05 | global_batch_size: 32 | global_step: 763 | reduced_train_loss: 0.9963 | train_step_timing in s: 2.402 | consumed_samples: 24448 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 764/999 | lr: 1.447e-05 | global_batch_size: 32 | global_step: 764 | reduced_train_loss: 1.2 | train_step_timing in s: 2.354 | consumed_samples: 24480 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 765/999 | lr: 1.435e-05 | global_batch_size: 32 | global_step: 765 | reduced_train_loss: 1.327 | train_step_timing in s: 2.418 | consumed_samples: 24512 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 766/999 | lr: 1.424e-05 | global_batch_size: 32 | global_step: 766 | reduced_train_loss: 1.314 | train_step_timing in s: 2.404 | consumed_samples: 24544 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 767/999 | lr: 1.412e-05 | global_batch_size: 32 | global_step: 767 | reduced_train_loss: 1.005 | train_step_timing in s: 2.414 | consumed_samples: 24576 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 768/999 | lr: 1.401e-05 | global_batch_size: 32 | global_step: 768 | reduced_train_loss: 1.061 | train_step_timing in s: 2.415 | consumed_samples: 24608 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 769/999 | lr: 1.389e-05 | global_batch_size: 32 | global_step: 769 | reduced_train_loss: 1.304 | train_step_timing in s: 2.403 | consumed_samples: 24640 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 770/999 | lr: 1.378e-05 | global_batch_size: 32 | global_step: 770 | reduced_train_loss: 1.071 | train_step_timing in s: 2.473 | consumed_samples: 24672 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 771/999 | lr: 1.366e-05 | global_batch_size: 32 | global_step: 771 | reduced_train_loss: 1.198 | train_step_timing in s: 2.411 | consumed_samples: 24704 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 772/999 | lr: 1.355e-05 | global_batch_size: 32 | global_step: 772 | reduced_train_loss: 1.189 | train_step_timing in s: 2.423 | consumed_samples: 24736 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 773/999 | lr: 1.344e-05 | global_batch_size: 32 | global_step: 773 | reduced_train_loss: 1.168 | train_step_timing in s: 2.417 | consumed_samples: 24768 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 774/999 | lr: 1.333e-05 | global_batch_size: 32 | global_step: 774 | reduced_train_loss: 1.216 | train_step_timing in s: 2.376 | consumed_samples: 24800 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 775/999 | lr: 1.321e-05 | global_batch_size: 32 | global_step: 775 | reduced_train_loss: 1.063 | train_step_timing in s: 2.432 | consumed_samples: 24832 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 776/999 | lr: 1.31e-05 | global_batch_size: 32 | global_step: 776 | reduced_train_loss: 1.26 | train_step_timing in s: 2.396 | consumed_samples: 24864 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 777/999 | lr: 1.299e-05 | global_batch_size: 32 | global_step: 777 | reduced_train_loss: 1.1 | train_step_timing in s: 2.418 | consumed_samples: 24896 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 778/999 | lr: 1.288e-05 | global_batch_size: 32 | global_step: 778 | reduced_train_loss: 1.021 | train_step_timing in s: 2.436 | consumed_samples: 24928 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 779/999 | lr: 1.277e-05 | global_batch_size: 32 | global_step: 779 | reduced_train_loss: 1.131 | train_step_timing in s: 2.395 | consumed_samples: 24960 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:23 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 780 : Start time: 1764082462.971s : Save duration: 0.619s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:23 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=779-consumed_samples=24960.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 780/999 | lr: 1.266e-05 | global_batch_size: 32 | global_step: 780 | reduced_train_loss: 1.13 | train_step_timing in s: 2.772 | consumed_samples: 24992 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:26 pl:222] Successfully saved checkpoint from iteration     780 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=779-consumed_samples=24960.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:26 model_checkpoint:657] Async checkpoint save for step 780 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=779-consumed_samples=24960.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:26 dist_ckpt_io:160] Async finalization time took 0.125 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 781/999 | lr: 1.255e-05 | global_batch_size: 32 | global_step: 781 | reduced_train_loss: 1.124 | train_step_timing in s: 2.41 | consumed_samples: 25024 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 782/999 | lr: 1.244e-05 | global_batch_size: 32 | global_step: 782 | reduced_train_loss: 1.183 | train_step_timing in s: 2.394 | consumed_samples: 25056 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 783/999 | lr: 1.233e-05 | global_batch_size: 32 | global_step: 783 | reduced_train_loss: 1.257 | train_step_timing in s: 2.316 | consumed_samples: 25088 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 784/999 | lr: 1.222e-05 | global_batch_size: 32 | global_step: 784 | reduced_train_loss: 1.137 | train_step_timing in s: 2.403 | consumed_samples: 25120 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 785/999 | lr: 1.211e-05 | global_batch_size: 32 | global_step: 785 | reduced_train_loss: 0.8565 | train_step_timing in s: 2.361 | consumed_samples: 25152 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 786/999 | lr: 1.201e-05 | global_batch_size: 32 | global_step: 786 | reduced_train_loss: 1.112 | train_step_timing in s: 2.419 | consumed_samples: 25184 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 787/999 | lr: 1.19e-05 | global_batch_size: 32 | global_step: 787 | reduced_train_loss: 1.084 | train_step_timing in s: 2.409 | consumed_samples: 25216 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 788/999 | lr: 1.179e-05 | global_batch_size: 32 | global_step: 788 | reduced_train_loss: 1.153 | train_step_timing in s: 2.448 | consumed_samples: 25248 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 789/999 | lr: 1.169e-05 | global_batch_size: 32 | global_step: 789 | reduced_train_loss: 1.27 | train_step_timing in s: 2.398 | consumed_samples: 25280 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 790/999 | lr: 1.158e-05 | global_batch_size: 32 | global_step: 790 | reduced_train_loss: 1.154 | train_step_timing in s: 2.395 | consumed_samples: 25312 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 791/999 | lr: 1.147e-05 | global_batch_size: 32 | global_step: 791 | reduced_train_loss: 1.221 | train_step_timing in s: 2.346 | consumed_samples: 25344 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 792/999 | lr: 1.137e-05 | global_batch_size: 32 | global_step: 792 | reduced_train_loss: 1.016 | train_step_timing in s: 2.371 | consumed_samples: 25376 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 793/999 | lr: 1.126e-05 | global_batch_size: 32 | global_step: 793 | reduced_train_loss: 0.9885 | train_step_timing in s: 2.388 | consumed_samples: 25408 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 794/999 | lr: 1.116e-05 | global_batch_size: 32 | global_step: 794 | reduced_train_loss: 0.9983 | train_step_timing in s: 2.412 | consumed_samples: 25440 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 795/999 | lr: 1.106e-05 | global_batch_size: 32 | global_step: 795 | reduced_train_loss: 0.9859 | train_step_timing in s: 2.464 | consumed_samples: 25472 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 796/999 | lr: 1.095e-05 | global_batch_size: 32 | global_step: 796 | reduced_train_loss: 1.15 | train_step_timing in s: 2.525 | consumed_samples: 25504 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 797/999 | lr: 1.085e-05 | global_batch_size: 32 | global_step: 797 | reduced_train_loss: 1.27 | train_step_timing in s: 2.402 | consumed_samples: 25536 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 798/999 | lr: 1.075e-05 | global_batch_size: 32 | global_step: 798 | reduced_train_loss: 1.051 | train_step_timing in s: 2.36 | consumed_samples: 25568 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 799/999 | lr: 1.064e-05 | global_batch_size: 32 | global_step: 799 | reduced_train_loss: 1.081 | train_step_timing in s: 2.438 | consumed_samples: 25600 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 799: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:13 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 799 : Start time: 1764082512.934s : Save duration: 0.173s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:13 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=799-consumed_samples=25600.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:13 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 800/999 | lr: 1.054e-05 | global_batch_size: 32 | global_step: 800 | reduced_train_loss: 1.221 | train_step_timing in s: 2.714 | consumed_samples: 25632 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:16 pl:222] Successfully saved checkpoint from iteration     799 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=799-consumed_samples=25600.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:16 model_checkpoint:657] Async checkpoint save for step 800 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=799-consumed_samples=25600.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:16 dist_ckpt_io:160] Async finalization time took 0.125 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 801/999 | lr: 1.044e-05 | global_batch_size: 32 | global_step: 801 | reduced_train_loss: 1.12 | train_step_timing in s: 2.402 | consumed_samples: 25664 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 802/999 | lr: 1.034e-05 | global_batch_size: 32 | global_step: 802 | reduced_train_loss: 1.066 | train_step_timing in s: 2.469 | consumed_samples: 25696 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 803/999 | lr: 1.024e-05 | global_batch_size: 32 | global_step: 803 | reduced_train_loss: 1.22 | train_step_timing in s: 2.407 | consumed_samples: 25728 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 804/999 | lr: 1.014e-05 | global_batch_size: 32 | global_step: 804 | reduced_train_loss: 1.258 | train_step_timing in s: 2.337 | consumed_samples: 25760 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 805/999 | lr: 1.004e-05 | global_batch_size: 32 | global_step: 805 | reduced_train_loss: 1.2 | train_step_timing in s: 2.426 | consumed_samples: 25792 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 806/999 | lr: 9.941e-06 | global_batch_size: 32 | global_step: 806 | reduced_train_loss: 1.153 | train_step_timing in s: 2.399 | consumed_samples: 25824 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 807/999 | lr: 9.843e-06 | global_batch_size: 32 | global_step: 807 | reduced_train_loss: 0.9676 | train_step_timing in s: 2.398 | consumed_samples: 25856 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 808/999 | lr: 9.744e-06 | global_batch_size: 32 | global_step: 808 | reduced_train_loss: 1.027 | train_step_timing in s: 2.417 | consumed_samples: 25888 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 809/999 | lr: 9.647e-06 | global_batch_size: 32 | global_step: 809 | reduced_train_loss: 1.071 | train_step_timing in s: 2.481 | consumed_samples: 25920 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:56:35 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 810 : Start time: 1764082595.172s : Save duration: 0.168s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:56:35 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=809-consumed_samples=25920.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 810/999 | lr: 9.549e-06 | global_batch_size: 32 | global_step: 810 | reduced_train_loss: 1.133 | train_step_timing in s: 2.905 | consumed_samples: 25952 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:56:38 pl:222] Successfully saved checkpoint from iteration     810 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=809-consumed_samples=25920.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:56:38 model_checkpoint:657] Async checkpoint save for step 810 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=809-consumed_samples=25920.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:56:38 dist_ckpt_io:160] Async finalization time took 0.124 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 811/999 | lr: 9.452e-06 | global_batch_size: 32 | global_step: 811 | reduced_train_loss: 1.136 | train_step_timing in s: 2.401 | consumed_samples: 25984 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 812/999 | lr: 9.356e-06 | global_batch_size: 32 | global_step: 812 | reduced_train_loss: 1.17 | train_step_timing in s: 2.41 | consumed_samples: 26016 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 813/999 | lr: 9.26e-06 | global_batch_size: 32 | global_step: 813 | reduced_train_loss: 1.084 | train_step_timing in s: 2.391 | consumed_samples: 26048 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 814/999 | lr: 9.164e-06 | global_batch_size: 32 | global_step: 814 | reduced_train_loss: 1.154 | train_step_timing in s: 2.419 | consumed_samples: 26080 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 815/999 | lr: 9.069e-06 | global_batch_size: 32 | global_step: 815 | reduced_train_loss: 1.322 | train_step_timing in s: 2.347 | consumed_samples: 26112 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 816/999 | lr: 8.974e-06 | global_batch_size: 32 | global_step: 816 | reduced_train_loss: 1.15 | train_step_timing in s: 2.418 | consumed_samples: 26144 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 817/999 | lr: 8.88e-06 | global_batch_size: 32 | global_step: 817 | reduced_train_loss: 1.156 | train_step_timing in s: 2.442 | consumed_samples: 26176 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 818/999 | lr: 8.786e-06 | global_batch_size: 32 | global_step: 818 | reduced_train_loss: 1.262 | train_step_timing in s: 2.368 | consumed_samples: 26208 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 819/999 | lr: 8.692e-06 | global_batch_size: 32 | global_step: 819 | reduced_train_loss: 1.277 | train_step_timing in s: 2.454 | consumed_samples: 26240 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 820/999 | lr: 8.6e-06 | global_batch_size: 32 | global_step: 820 | reduced_train_loss: 1.034 | train_step_timing in s: 2.424 | consumed_samples: 26272 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 821/999 | lr: 8.507e-06 | global_batch_size: 32 | global_step: 821 | reduced_train_loss: 1.29 | train_step_timing in s: 2.491 | consumed_samples: 26304 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 822/999 | lr: 8.415e-06 | global_batch_size: 32 | global_step: 822 | reduced_train_loss: 1.128 | train_step_timing in s: 2.408 | consumed_samples: 26336 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 823/999 | lr: 8.323e-06 | global_batch_size: 32 | global_step: 823 | reduced_train_loss: 1.183 | train_step_timing in s: 2.448 | consumed_samples: 26368 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 824/999 | lr: 8.232e-06 | global_batch_size: 32 | global_step: 824 | reduced_train_loss: 1.253 | train_step_timing in s: 2.415 | consumed_samples: 26400 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 825/999 | lr: 8.142e-06 | global_batch_size: 32 | global_step: 825 | reduced_train_loss: 1.197 | train_step_timing in s: 2.426 | consumed_samples: 26432 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 826/999 | lr: 8.051e-06 | global_batch_size: 32 | global_step: 826 | reduced_train_loss: 1.03 | train_step_timing in s: 2.432 | consumed_samples: 26464 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 827/999 | lr: 7.962e-06 | global_batch_size: 32 | global_step: 827 | reduced_train_loss: 1.287 | train_step_timing in s: 2.418 | consumed_samples: 26496 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 828/999 | lr: 7.872e-06 | global_batch_size: 32 | global_step: 828 | reduced_train_loss: 1.238 | train_step_timing in s: 2.426 | consumed_samples: 26528 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 829/999 | lr: 7.784e-06 | global_batch_size: 32 | global_step: 829 | reduced_train_loss: 1.194 | train_step_timing in s: 2.4 | consumed_samples: 26560 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 830/999 | lr: 7.695e-06 | global_batch_size: 32 | global_step: 830 | reduced_train_loss: 1.057 | train_step_timing in s: 2.424 | consumed_samples: 26592 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 831/999 | lr: 7.607e-06 | global_batch_size: 32 | global_step: 831 | reduced_train_loss: 1.278 | train_step_timing in s: 2.444 | consumed_samples: 26624 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 832/999 | lr: 7.52e-06 | global_batch_size: 32 | global_step: 832 | reduced_train_loss: 1.072 | train_step_timing in s: 2.476 | consumed_samples: 26656 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 833/999 | lr: 7.433e-06 | global_batch_size: 32 | global_step: 833 | reduced_train_loss: 1.133 | train_step_timing in s: 2.425 | consumed_samples: 26688 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 834/999 | lr: 7.346e-06 | global_batch_size: 32 | global_step: 834 | reduced_train_loss: 1.024 | train_step_timing in s: 2.448 | consumed_samples: 26720 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 835/999 | lr: 7.26e-06 | global_batch_size: 32 | global_step: 835 | reduced_train_loss: 1.121 | train_step_timing in s: 2.422 | consumed_samples: 26752 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 836/999 | lr: 7.175e-06 | global_batch_size: 32 | global_step: 836 | reduced_train_loss: 1.117 | train_step_timing in s: 2.5 | consumed_samples: 26784 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 837/999 | lr: 7.09e-06 | global_batch_size: 32 | global_step: 837 | reduced_train_loss: 1.004 | train_step_timing in s: 2.407 | consumed_samples: 26816 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 838/999 | lr: 7.005e-06 | global_batch_size: 32 | global_step: 838 | reduced_train_loss: 1.306 | train_step_timing in s: 2.389 | consumed_samples: 26848 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 839/999 | lr: 6.921e-06 | global_batch_size: 32 | global_step: 839 | reduced_train_loss: 1.089 | train_step_timing in s: 2.336 | consumed_samples: 26880 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:46 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 840 : Start time: 1764082726.571s : Save duration: 0.174s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:47 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=839-consumed_samples=26880.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 840/999 | lr: 6.837e-06 | global_batch_size: 32 | global_step: 840 | reduced_train_loss: 1.279 | train_step_timing in s: 2.763 | consumed_samples: 26912 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:50 pl:222] Successfully saved checkpoint from iteration     840 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=839-consumed_samples=26880.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:50 model_checkpoint:657] Async checkpoint save for step 840 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=839-consumed_samples=26880.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:50 dist_ckpt_io:160] Async finalization time took 0.558 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 841/999 | lr: 6.754e-06 | global_batch_size: 32 | global_step: 841 | reduced_train_loss: 1.089 | train_step_timing in s: 2.385 | consumed_samples: 26944 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 842/999 | lr: 6.671e-06 | global_batch_size: 32 | global_step: 842 | reduced_train_loss: 1.007 | train_step_timing in s: 2.385 | consumed_samples: 26976 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 843/999 | lr: 6.589e-06 | global_batch_size: 32 | global_step: 843 | reduced_train_loss: 1.15 | train_step_timing in s: 2.427 | consumed_samples: 27008 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 844/999 | lr: 6.507e-06 | global_batch_size: 32 | global_step: 844 | reduced_train_loss: 1.026 | train_step_timing in s: 2.439 | consumed_samples: 27040 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 845/999 | lr: 6.426e-06 | global_batch_size: 32 | global_step: 845 | reduced_train_loss: 1.156 | train_step_timing in s: 2.408 | consumed_samples: 27072 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 846/999 | lr: 6.345e-06 | global_batch_size: 32 | global_step: 846 | reduced_train_loss: 1.093 | train_step_timing in s: 2.43 | consumed_samples: 27104 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 847/999 | lr: 6.265e-06 | global_batch_size: 32 | global_step: 847 | reduced_train_loss: 1.208 | train_step_timing in s: 2.381 | consumed_samples: 27136 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 848/999 | lr: 6.185e-06 | global_batch_size: 32 | global_step: 848 | reduced_train_loss: 1.046 | train_step_timing in s: 2.399 | consumed_samples: 27168 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 849/999 | lr: 6.105e-06 | global_batch_size: 32 | global_step: 849 | reduced_train_loss: 1.24 | train_step_timing in s: 2.351 | consumed_samples: 27200 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 849: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:12 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 849 : Start time: 1764082752.298s : Save duration: 0.168s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:12 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=849-consumed_samples=27200.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:12 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 850/999 | lr: 6.026e-06 | global_batch_size: 32 | global_step: 850 | reduced_train_loss: 0.9417 | train_step_timing in s: 2.713 | consumed_samples: 27232 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:15 pl:222] Successfully saved checkpoint from iteration     849 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=849-consumed_samples=27200.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:15 model_checkpoint:657] Async checkpoint save for step 850 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=849-consumed_samples=27200.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:15 dist_ckpt_io:160] Async finalization time took 0.123 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 851/999 | lr: 5.948e-06 | global_batch_size: 32 | global_step: 851 | reduced_train_loss: 1.322 | train_step_timing in s: 2.347 | consumed_samples: 27264 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 852/999 | lr: 5.87e-06 | global_batch_size: 32 | global_step: 852 | reduced_train_loss: 1.128 | train_step_timing in s: 2.459 | consumed_samples: 27296 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 853/999 | lr: 5.792e-06 | global_batch_size: 32 | global_step: 853 | reduced_train_loss: 1.09 | train_step_timing in s: 2.376 | consumed_samples: 27328 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 854/999 | lr: 5.715e-06 | global_batch_size: 32 | global_step: 854 | reduced_train_loss: 1.146 | train_step_timing in s: 2.371 | consumed_samples: 27360 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 855/999 | lr: 5.639e-06 | global_batch_size: 32 | global_step: 855 | reduced_train_loss: 1.136 | train_step_timing in s: 2.409 | consumed_samples: 27392 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 856/999 | lr: 5.563e-06 | global_batch_size: 32 | global_step: 856 | reduced_train_loss: 1.206 | train_step_timing in s: 2.423 | consumed_samples: 27424 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 857/999 | lr: 5.487e-06 | global_batch_size: 32 | global_step: 857 | reduced_train_loss: 1.107 | train_step_timing in s: 2.427 | consumed_samples: 27456 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 858/999 | lr: 5.412e-06 | global_batch_size: 32 | global_step: 858 | reduced_train_loss: 1.204 | train_step_timing in s: 2.37 | consumed_samples: 27488 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 859/999 | lr: 5.338e-06 | global_batch_size: 32 | global_step: 859 | reduced_train_loss: 1.089 | train_step_timing in s: 2.402 | consumed_samples: 27520 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 860/999 | lr: 5.264e-06 | global_batch_size: 32 | global_step: 860 | reduced_train_loss: 1.13 | train_step_timing in s: 2.403 | consumed_samples: 27552 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 861/999 | lr: 5.19e-06 | global_batch_size: 32 | global_step: 861 | reduced_train_loss: 0.9514 | train_step_timing in s: 2.41 | consumed_samples: 27584 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 862/999 | lr: 5.117e-06 | global_batch_size: 32 | global_step: 862 | reduced_train_loss: 1.078 | train_step_timing in s: 2.404 | consumed_samples: 27616 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 863/999 | lr: 5.044e-06 | global_batch_size: 32 | global_step: 863 | reduced_train_loss: 1.191 | train_step_timing in s: 2.423 | consumed_samples: 27648 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 864/999 | lr: 4.972e-06 | global_batch_size: 32 | global_step: 864 | reduced_train_loss: 1.118 | train_step_timing in s: 2.391 | consumed_samples: 27680 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 865/999 | lr: 4.9e-06 | global_batch_size: 32 | global_step: 865 | reduced_train_loss: 1.376 | train_step_timing in s: 2.401 | consumed_samples: 27712 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 866/999 | lr: 4.829e-06 | global_batch_size: 32 | global_step: 866 | reduced_train_loss: 1.101 | train_step_timing in s: 2.4 | consumed_samples: 27744 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 867/999 | lr: 4.759e-06 | global_batch_size: 32 | global_step: 867 | reduced_train_loss: 1.226 | train_step_timing in s: 2.375 | consumed_samples: 27776 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 868/999 | lr: 4.688e-06 | global_batch_size: 32 | global_step: 868 | reduced_train_loss: 1.238 | train_step_timing in s: 2.403 | consumed_samples: 27808 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 869/999 | lr: 4.619e-06 | global_batch_size: 32 | global_step: 869 | reduced_train_loss: 1.148 | train_step_timing in s: 2.387 | consumed_samples: 27840 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:00:59 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 870 : Start time: 1764082859.241s : Save duration: 0.168s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:00:59 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=869-consumed_samples=27840.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 870/999 | lr: 4.55e-06 | global_batch_size: 32 | global_step: 870 | reduced_train_loss: 1.141 | train_step_timing in s: 2.788 | consumed_samples: 27872 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:01:02 pl:222] Successfully saved checkpoint from iteration     870 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=869-consumed_samples=27840.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:01:02 model_checkpoint:657] Async checkpoint save for step 870 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=869-consumed_samples=27840.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:01:02 dist_ckpt_io:160] Async finalization time took 0.127 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 871/999 | lr: 4.481e-06 | global_batch_size: 32 | global_step: 871 | reduced_train_loss: 1.183 | train_step_timing in s: 2.43 | consumed_samples: 27904 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 872/999 | lr: 4.413e-06 | global_batch_size: 32 | global_step: 872 | reduced_train_loss: 1.08 | train_step_timing in s: 2.415 | consumed_samples: 27936 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 873/999 | lr: 4.345e-06 | global_batch_size: 32 | global_step: 873 | reduced_train_loss: 1.308 | train_step_timing in s: 2.415 | consumed_samples: 27968 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 874/999 | lr: 4.278e-06 | global_batch_size: 32 | global_step: 874 | reduced_train_loss: 0.9828 | train_step_timing in s: 2.384 | consumed_samples: 28000 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 875/999 | lr: 4.211e-06 | global_batch_size: 32 | global_step: 875 | reduced_train_loss: 1.016 | train_step_timing in s: 2.365 | consumed_samples: 28032 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 876/999 | lr: 4.145e-06 | global_batch_size: 32 | global_step: 876 | reduced_train_loss: 1.093 | train_step_timing in s: 2.355 | consumed_samples: 28064 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 877/999 | lr: 4.079e-06 | global_batch_size: 32 | global_step: 877 | reduced_train_loss: 1.158 | train_step_timing in s: 2.397 | consumed_samples: 28096 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 878/999 | lr: 4.014e-06 | global_batch_size: 32 | global_step: 878 | reduced_train_loss: 1.08 | train_step_timing in s: 2.511 | consumed_samples: 28128 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 879/999 | lr: 3.95e-06 | global_batch_size: 32 | global_step: 879 | reduced_train_loss: 1.282 | train_step_timing in s: 2.395 | consumed_samples: 28160 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 880/999 | lr: 3.886e-06 | global_batch_size: 32 | global_step: 880 | reduced_train_loss: 0.9977 | train_step_timing in s: 2.475 | consumed_samples: 28192 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 881/999 | lr: 3.822e-06 | global_batch_size: 32 | global_step: 881 | reduced_train_loss: 1.134 | train_step_timing in s: 2.484 | consumed_samples: 28224 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 882/999 | lr: 3.759e-06 | global_batch_size: 32 | global_step: 882 | reduced_train_loss: 1.063 | train_step_timing in s: 2.416 | consumed_samples: 28256 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 883/999 | lr: 3.696e-06 | global_batch_size: 32 | global_step: 883 | reduced_train_loss: 1.132 | train_step_timing in s: 2.413 | consumed_samples: 28288 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 884/999 | lr: 3.634e-06 | global_batch_size: 32 | global_step: 884 | reduced_train_loss: 1.114 | train_step_timing in s: 2.387 | consumed_samples: 28320 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 885/999 | lr: 3.572e-06 | global_batch_size: 32 | global_step: 885 | reduced_train_loss: 1.063 | train_step_timing in s: 2.371 | consumed_samples: 28352 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 886/999 | lr: 3.511e-06 | global_batch_size: 32 | global_step: 886 | reduced_train_loss: 1.244 | train_step_timing in s: 2.374 | consumed_samples: 28384 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 887/999 | lr: 3.451e-06 | global_batch_size: 32 | global_step: 887 | reduced_train_loss: 1.043 | train_step_timing in s: 2.474 | consumed_samples: 28416 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 888/999 | lr: 3.39e-06 | global_batch_size: 32 | global_step: 888 | reduced_train_loss: 1.121 | train_step_timing in s: 2.407 | consumed_samples: 28448 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 889/999 | lr: 3.331e-06 | global_batch_size: 32 | global_step: 889 | reduced_train_loss: 1.025 | train_step_timing in s: 2.416 | consumed_samples: 28480 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 890/999 | lr: 3.272e-06 | global_batch_size: 32 | global_step: 890 | reduced_train_loss: 1.085 | train_step_timing in s: 2.395 | consumed_samples: 28512 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 891/999 | lr: 3.213e-06 | global_batch_size: 32 | global_step: 891 | reduced_train_loss: 1.124 | train_step_timing in s: 2.385 | consumed_samples: 28544 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 892/999 | lr: 3.155e-06 | global_batch_size: 32 | global_step: 892 | reduced_train_loss: 1.25 | train_step_timing in s: 2.405 | consumed_samples: 28576 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 893/999 | lr: 3.098e-06 | global_batch_size: 32 | global_step: 893 | reduced_train_loss: 1.105 | train_step_timing in s: 2.435 | consumed_samples: 28608 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 894/999 | lr: 3.041e-06 | global_batch_size: 32 | global_step: 894 | reduced_train_loss: 1.161 | train_step_timing in s: 2.419 | consumed_samples: 28640 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 895/999 | lr: 2.984e-06 | global_batch_size: 32 | global_step: 895 | reduced_train_loss: 1.245 | train_step_timing in s: 2.404 | consumed_samples: 28672 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 896/999 | lr: 2.928e-06 | global_batch_size: 32 | global_step: 896 | reduced_train_loss: 1.14 | train_step_timing in s: 2.47 | consumed_samples: 28704 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 897/999 | lr: 2.873e-06 | global_batch_size: 32 | global_step: 897 | reduced_train_loss: 1.199 | train_step_timing in s: 2.377 | consumed_samples: 28736 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 898/999 | lr: 2.818e-06 | global_batch_size: 32 | global_step: 898 | reduced_train_loss: 1.013 | train_step_timing in s: 2.421 | consumed_samples: 28768 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 899/999 | lr: 2.763e-06 | global_batch_size: 32 | global_step: 899 | reduced_train_loss: 1.154 | train_step_timing in s: 2.46 | consumed_samples: 28800 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 899: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:02:13 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 899 : Start time: 1764082933.372s : Save duration: 0.176s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:02:13 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=899-consumed_samples=28800.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:02:13 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:Training epoch 0, iteration 900/999 | lr: 2.709e-06 | global_batch_size: 32 | global_step: 900 | reduced_train_loss: 0.9141 | train_step_timing in s: 2.518 | consumed_samples: 28832 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:14 pl:222] Successfully saved checkpoint from iteration     899 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=899-consumed_samples=28800.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:14 model_checkpoint:657] Async checkpoint save for step 900 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=899-consumed_samples=28800.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:14 dist_ckpt_io:160] Async finalization time took 0.140 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 901/999 | lr: 2.656e-06 | global_batch_size: 32 | global_step: 901 | reduced_train_loss: 1.02 | train_step_timing in s: 2.514 | consumed_samples: 28864 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 902/999 | lr: 2.603e-06 | global_batch_size: 32 | global_step: 902 | reduced_train_loss: 1.059 | train_step_timing in s: 2.422 | consumed_samples: 28896 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 903/999 | lr: 2.55e-06 | global_batch_size: 32 | global_step: 903 | reduced_train_loss: 0.9627 | train_step_timing in s: 2.462 | consumed_samples: 28928 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 904/999 | lr: 2.499e-06 | global_batch_size: 32 | global_step: 904 | reduced_train_loss: 1.059 | train_step_timing in s: 2.417 | consumed_samples: 28960 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 905/999 | lr: 2.447e-06 | global_batch_size: 32 | global_step: 905 | reduced_train_loss: 1.047 | train_step_timing in s: 2.467 | consumed_samples: 28992 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 906/999 | lr: 2.396e-06 | global_batch_size: 32 | global_step: 906 | reduced_train_loss: 1.076 | train_step_timing in s: 2.364 | consumed_samples: 29024 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 907/999 | lr: 2.346e-06 | global_batch_size: 32 | global_step: 907 | reduced_train_loss: 1.056 | train_step_timing in s: 2.394 | consumed_samples: 29056 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 908/999 | lr: 2.296e-06 | global_batch_size: 32 | global_step: 908 | reduced_train_loss: 1.171 | train_step_timing in s: 2.476 | consumed_samples: 29088 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 909/999 | lr: 2.247e-06 | global_batch_size: 32 | global_step: 909 | reduced_train_loss: 1.163 | train_step_timing in s: 2.39 | consumed_samples: 29120 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 910/999 | lr: 2.198e-06 | global_batch_size: 32 | global_step: 910 | reduced_train_loss: 1.078 | train_step_timing in s: 2.442 | consumed_samples: 29152 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 911/999 | lr: 2.15e-06 | global_batch_size: 32 | global_step: 911 | reduced_train_loss: 1.171 | train_step_timing in s: 2.405 | consumed_samples: 29184 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 912/999 | lr: 2.102e-06 | global_batch_size: 32 | global_step: 912 | reduced_train_loss: 1.258 | train_step_timing in s: 2.375 | consumed_samples: 29216 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 913/999 | lr: 2.055e-06 | global_batch_size: 32 | global_step: 913 | reduced_train_loss: 1.053 | train_step_timing in s: 2.388 | consumed_samples: 29248 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 914/999 | lr: 2.008e-06 | global_batch_size: 32 | global_step: 914 | reduced_train_loss: 1.061 | train_step_timing in s: 2.41 | consumed_samples: 29280 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 915/999 | lr: 1.962e-06 | global_batch_size: 32 | global_step: 915 | reduced_train_loss: 1.162 | train_step_timing in s: 2.403 | consumed_samples: 29312 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 916/999 | lr: 1.917e-06 | global_batch_size: 32 | global_step: 916 | reduced_train_loss: 0.9781 | train_step_timing in s: 2.428 | consumed_samples: 29344 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 917/999 | lr: 1.872e-06 | global_batch_size: 32 | global_step: 917 | reduced_train_loss: 1.22 | train_step_timing in s: 2.358 | consumed_samples: 29376 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 918/999 | lr: 1.827e-06 | global_batch_size: 32 | global_step: 918 | reduced_train_loss: 1.137 | train_step_timing in s: 2.403 | consumed_samples: 29408 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 919/999 | lr: 1.783e-06 | global_batch_size: 32 | global_step: 919 | reduced_train_loss: 1.26 | train_step_timing in s: 2.464 | consumed_samples: 29440 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 920/999 | lr: 1.74e-06 | global_batch_size: 32 | global_step: 920 | reduced_train_loss: 1.196 | train_step_timing in s: 2.414 | consumed_samples: 29472 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 921/999 | lr: 1.697e-06 | global_batch_size: 32 | global_step: 921 | reduced_train_loss: 1.142 | train_step_timing in s: 2.381 | consumed_samples: 29504 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 922/999 | lr: 1.654e-06 | global_batch_size: 32 | global_step: 922 | reduced_train_loss: 1.169 | train_step_timing in s: 2.381 | consumed_samples: 29536 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 923/999 | lr: 1.612e-06 | global_batch_size: 32 | global_step: 923 | reduced_train_loss: 1.21 | train_step_timing in s: 2.46 | consumed_samples: 29568 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 924/999 | lr: 1.571e-06 | global_batch_size: 32 | global_step: 924 | reduced_train_loss: 1.133 | train_step_timing in s: 2.442 | consumed_samples: 29600 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 925/999 | lr: 1.53e-06 | global_batch_size: 32 | global_step: 925 | reduced_train_loss: 0.9718 | train_step_timing in s: 2.379 | consumed_samples: 29632 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 926/999 | lr: 1.49e-06 | global_batch_size: 32 | global_step: 926 | reduced_train_loss: 1.12 | train_step_timing in s: 2.431 | consumed_samples: 29664 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 927/999 | lr: 1.45e-06 | global_batch_size: 32 | global_step: 927 | reduced_train_loss: 1.266 | train_step_timing in s: 2.402 | consumed_samples: 29696 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 928/999 | lr: 1.411e-06 | global_batch_size: 32 | global_step: 928 | reduced_train_loss: 1.158 | train_step_timing in s: 2.43 | consumed_samples: 29728 | val_loss: 1.165[default0]:
s_act_ckpt/0 [default0]:Training epoch 0, iteration 929/999 | lr: 1.372e-06 | global_batch_size: 32 | global_step: 929 | reduced_train_loss: 1.11 | train_step_timing in s: 2.435 | consumed_samples: 29760 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:22 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 930 : Start time: 1764083122.310s : Save duration: 0.628s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:23 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=929-consumed_samples=29760.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 930/999 | lr: 1.334e-06 | global_batch_size: 32 | global_step: 930 | reduced_train_loss: 1.14 | train_step_timing in s: 2.804 | consumed_samples: 29792 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:26 pl:222] Successfully saved checkpoint from iteration     930 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=929-consumed_samples=29760.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:26 model_checkpoint:657] Async checkpoint save for step 930 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=929-consumed_samples=29760.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:26 dist_ckpt_io:160] Async finalization time took 0.128 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 931/999 | lr: 1.296e-06 | global_batch_size: 32 | global_step: 931 | reduced_train_loss: 1.163 | train_step_timing in s: 2.467 | consumed_samples: 29824 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 932/999 | lr: 1.259e-06 | global_batch_size: 32 | global_step: 932 | reduced_train_loss: 1.179 | train_step_timing in s: 2.495 | consumed_samples: 29856 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 933/999 | lr: 1.222e-06 | global_batch_size: 32 | global_step: 933 | reduced_train_loss: 1.04 | train_step_timing in s: 2.457 | consumed_samples: 29888 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 934/999 | lr: 1.186e-06 | global_batch_size: 32 | global_step: 934 | reduced_train_loss: 0.9553 | train_step_timing in s: 2.404 | consumed_samples: 29920 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 935/999 | lr: 1.151e-06 | global_batch_size: 32 | global_step: 935 | reduced_train_loss: 1.069 | train_step_timing in s: 2.414 | consumed_samples: 29952 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 936/999 | lr: 1.116e-06 | global_batch_size: 32 | global_step: 936 | reduced_train_loss: 1.038 | train_step_timing in s: 2.459 | consumed_samples: 29984 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 937/999 | lr: 1.081e-06 | global_batch_size: 32 | global_step: 937 | reduced_train_loss: 0.9938 | train_step_timing in s: 2.496 | consumed_samples: 30016 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 938/999 | lr: 1.047e-06 | global_batch_size: 32 | global_step: 938 | reduced_train_loss: 1.017 | train_step_timing in s: 2.434 | consumed_samples: 30048 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 939/999 | lr: 1.014e-06 | global_batch_size: 32 | global_step: 939 | reduced_train_loss: 1.164 | train_step_timing in s: 2.446 | consumed_samples: 30080 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 940/999 | lr: 9.81e-07 | global_batch_size: 32 | global_step: 940 | reduced_train_loss: 1.121 | train_step_timing in s: 2.451 | consumed_samples: 30112 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 941/999 | lr: 9.487e-07 | global_batch_size: 32 | global_step: 941 | reduced_train_loss: 1.062 | train_step_timing in s: 2.436 | consumed_samples: 30144 | val_loss: 1.165[default0]:
s_act_ckpt/0 [default0]:Training epoch 0, iteration 942/999 | lr: 9.169e-07 | global_batch_size: 32 | global_step: 942 | reduced_train_loss: 1.059 | train_step_timing in s: 2.423 | consumed_samples: 30176 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 943/999 | lr: 8.856e-07 | global_batch_size: 32 | global_step: 943 | reduced_train_loss: 1.117 | train_step_timing in s: 2.377 | consumed_samples: 30208 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 944/999 | lr: 8.549e-07 | global_batch_size: 32 | global_step: 944 | reduced_train_loss: 1.104 | train_step_timing in s: 2.393 | consumed_samples: 30240 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 945/999 | lr: 8.247e-07 | global_batch_size: 32 | global_step: 945 | reduced_train_loss: 1.118 | train_step_timing in s: 2.442 | consumed_samples: 30272 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 946/999 | lr: 7.951e-07 | global_batch_size: 32 | global_step: 946 | reduced_train_loss: 1.047 | train_step_timing in s: 2.451 | consumed_samples: 30304 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 947/999 | lr: 7.66e-07 | global_batch_size: 32 | global_step: 947 | reduced_train_loss: 1.095 | train_step_timing in s: 2.482 | consumed_samples: 30336 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 948/999 | lr: 7.374e-07 | global_batch_size: 32 | global_step: 948 | reduced_train_loss: 1.018 | train_step_timing in s: 2.4 | consumed_samples: 30368 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 949/999 | lr: 7.094e-07 | global_batch_size: 32 | global_step: 949 | reduced_train_loss: 1.141 | train_step_timing in s: 2.36 | consumed_samples: 30400 | val_loss: 1.165
s_act_ckpt/0 [default0]:Epoch 0, global step 949: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:06:13 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 949 : Start time: 1764083172.907s : Save duration: 0.168s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:06:13 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=949-consumed_samples=30400.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:06:13 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 950/999 | lr: 6.819e-07 | global_batch_size: 32 | global_step: 950 | reduced_train_loss: 1.002 | train_step_timing in s: 2.9 | consumed_samples: 30432 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:06:16 pl:222] Successfully saved checkpoint from iteration     949 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=949-consumed_samples=30400.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:06:16 model_checkpoint:657] Async checkpoint save for step 950 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=949-consumed_samples=30400.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:06:16 dist_ckpt_io:160] Async finalization time took 0.125 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 951/999 | lr: 6.55e-07 | global_batch_size: 32 | global_step: 951 | reduced_train_loss: 1.072 | train_step_timing in s: 2.496 | consumed_samples: 30464 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 952/999 | lr: 6.286e-07 | global_batch_size: 32 | global_step: 952 | reduced_train_loss: 1.01 | train_step_timing in s: 2.45 | consumed_samples: 30496 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 953/999 | lr: 6.027e-07 | global_batch_size: 32 | global_step: 953 | reduced_train_loss: 1.099 | train_step_timing in s: 2.384 | consumed_samples: 30528 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 954/999 | lr: 5.774e-07 | global_batch_size: 32 | global_step: 954 | reduced_train_loss: 1.071 | train_step_timing in s: 2.441 | consumed_samples: 30560 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 955/999 | lr: 5.526e-07 | global_batch_size: 32 | global_step: 955 | reduced_train_loss: 1.396 | train_step_timing in s: 2.4 | consumed_samples: 30592 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 956/999 | lr: 5.284e-07 | global_batch_size: 32 | global_step: 956 | reduced_train_loss: 1.263 | train_step_timing in s: 2.357 | consumed_samples: 30624 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 957/999 | lr: 5.047e-07 | global_batch_size: 32 | global_step: 957 | reduced_train_loss: 1.503 | train_step_timing in s: 2.348 | consumed_samples: 30656 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 958/999 | lr: 4.815e-07 | global_batch_size: 32 | global_step: 958 | reduced_train_loss: 1.1 | train_step_timing in s: 2.369 | consumed_samples: 30688 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 959/999 | lr: 4.589e-07 | global_batch_size: 32 | global_step: 959 | reduced_train_loss: 1.047 | train_step_timing in s: 2.41 | consumed_samples: 30720 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:35 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 960 : Start time: 1764083255.091s : Save duration: 0.171s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:35 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=959-consumed_samples=30720.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 960/999 | lr: 4.368e-07 | global_batch_size: 32 | global_step: 960 | reduced_train_loss: 1.151 | train_step_timing in s: 2.743 | consumed_samples: 30752 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:38 pl:222] Successfully saved checkpoint from iteration     960 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=959-consumed_samples=30720.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:38 model_checkpoint:657] Async checkpoint save for step 960 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=959-consumed_samples=30720.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:38 dist_ckpt_io:160] Async finalization time took 0.132 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 961/999 | lr: 4.153e-07 | global_batch_size: 32 | global_step: 961 | reduced_train_loss: 1.29 | train_step_timing in s: 2.419 | consumed_samples: 30784 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 962/999 | lr: 3.943e-07 | global_batch_size: 32 | global_step: 962 | reduced_train_loss: 1.167 | train_step_timing in s: 2.374 | consumed_samples: 30816 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 963/999 | lr: 3.738e-07 | global_batch_size: 32 | global_step: 963 | reduced_train_loss: 1.103 | train_step_timing in s: 2.417 | consumed_samples: 30848 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 964/999 | lr: 3.539e-07 | global_batch_size: 32 | global_step: 964 | reduced_train_loss: 1.096 | train_step_timing in s: 2.324 | consumed_samples: 30880 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 965/999 | lr: 3.345e-07 | global_batch_size: 32 | global_step: 965 | reduced_train_loss: 1.15 | train_step_timing in s: 2.343 | consumed_samples: 30912 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 966/999 | lr: 3.157e-07 | global_batch_size: 32 | global_step: 966 | reduced_train_loss: 1.061 | train_step_timing in s: 2.371 | consumed_samples: 30944 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 967/999 | lr: 2.974e-07 | global_batch_size: 32 | global_step: 967 | reduced_train_loss: 1.183 | train_step_timing in s: 2.374 | consumed_samples: 30976 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 968/999 | lr: 2.797e-07 | global_batch_size: 32 | global_step: 968 | reduced_train_loss: 1.107 | train_step_timing in s: 2.344 | consumed_samples: 31008 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 969/999 | lr: 2.625e-07 | global_batch_size: 32 | global_step: 969 | reduced_train_loss: 1.214 | train_step_timing in s: 2.354 | consumed_samples: 31040 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 970/999 | lr: 2.459e-07 | global_batch_size: 32 | global_step: 970 | reduced_train_loss: 1.127 | train_step_timing in s: 2.422 | consumed_samples: 31072 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 971/999 | lr: 2.298e-07 | global_batch_size: 32 | global_step: 971 | reduced_train_loss: 1.019 | train_step_timing in s: 2.431 | consumed_samples: 31104 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 972/999 | lr: 2.142e-07 | global_batch_size: 32 | global_step: 972 | reduced_train_loss: 1.135 | train_step_timing in s: 2.368 | consumed_samples: 31136 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 973/999 | lr: 1.992e-07 | global_batch_size: 32 | global_step: 973 | reduced_train_loss: 1.101 | train_step_timing in s: 2.346 | consumed_samples: 31168 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 974/999 | lr: 1.847e-07 | global_batch_size: 32 | global_step: 974 | reduced_train_loss: 1.187 | train_step_timing in s: 2.458 | consumed_samples: 31200 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 975/999 | lr: 1.708e-07 | global_batch_size: 32 | global_step: 975 | reduced_train_loss: 1.158 | train_step_timing in s: 2.448 | consumed_samples: 31232 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 976/999 | lr: 1.574e-07 | global_batch_size: 32 | global_step: 976 | reduced_train_loss: 1.205 | train_step_timing in s: 2.398 | consumed_samples: 31264 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 977/999 | lr: 1.446e-07 | global_batch_size: 32 | global_step: 977 | reduced_train_loss: 1.109 | train_step_timing in s: 2.353 | consumed_samples: 31296 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 978/999 | lr: 1.323e-07 | global_batch_size: 32 | global_step: 978 | reduced_train_loss: 1.246 | train_step_timing in s: 2.335 | consumed_samples: 31328 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 979/999 | lr: 1.205e-07 | global_batch_size: 32 | global_step: 979 | reduced_train_loss: 1.072 | train_step_timing in s: 2.355 | consumed_samples: 31360 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 980/999 | lr: 1.093e-07 | global_batch_size: 32 | global_step: 980 | reduced_train_loss: 1.104 | train_step_timing in s: 2.38 | consumed_samples: 31392 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 981/999 | lr: 9.866e-08 | global_batch_size: 32 | global_step: 981 | reduced_train_loss: 1.045 | train_step_timing in s: 2.44 | consumed_samples: 31424 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 982/999 | lr: 8.855e-08 | global_batch_size: 32 | global_step: 982 | reduced_train_loss: 1.141 | train_step_timing in s: 2.409 | consumed_samples: 31456 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 983/999 | lr: 7.899e-08 | global_batch_size: 32 | global_step: 983 | reduced_train_loss: 1.15 | train_step_timing in s: 2.421 | consumed_samples: 31488 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 984/999 | lr: 6.997e-08 | global_batch_size: 32 | global_step: 984 | reduced_train_loss: 1.068 | train_step_timing in s: 2.418 | consumed_samples: 31520 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 985/999 | lr: 6.15e-08 | global_batch_size: 32 | global_step: 985 | reduced_train_loss: 1.129 | train_step_timing in s: 2.387 | consumed_samples: 31552 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 986/999 | lr: 5.358e-08 | global_batch_size: 32 | global_step: 986 | reduced_train_loss: 1.23 | train_step_timing in s: 2.413 | consumed_samples: 31584 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 987/999 | lr: 4.62e-08 | global_batch_size: 32 | global_step: 987 | reduced_train_loss: 1.284 | train_step_timing in s: 2.382 | consumed_samples: 31616 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 988/999 | lr: 3.936e-08 | global_batch_size: 32 | global_step: 988 | reduced_train_loss: 1.041 | train_step_timing in s: 2.387 | consumed_samples: 31648 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 989/999 | lr: 3.308e-08 | global_batch_size: 32 | global_step: 989 | reduced_train_loss: 1.253 | train_step_timing in s: 2.472 | consumed_samples: 31680 | val_loss: 1.165
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:45 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 990 : Start time: 1764083385.358s : Save duration: 0.173s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:46 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=989-consumed_samples=31680.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 990/999 | lr: 2.734e-08 | global_batch_size: 32 | global_step: 990 | reduced_train_loss: 1.205 | train_step_timing in s: 2.435 | consumed_samples: 31712 | val_loss: 1.165[default0]:
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:48 pl:222] Successfully saved checkpoint from iteration     990 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=989-consumed_samples=31680.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:48 model_checkpoint:657] Async checkpoint save for step 990 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=989-consumed_samples=31680.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:48 dist_ckpt_io:160] Async finalization time took 0.122 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 991/999 | lr: 2.214e-08 | global_batch_size: 32 | global_step: 991 | reduced_train_loss: 1.126 | train_step_timing in s: 2.431 | consumed_samples: 31744 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 992/999 | lr: 1.75e-08 | global_batch_size: 32 | global_step: 992 | reduced_train_loss: 1.106 | train_step_timing in s: 2.379 | consumed_samples: 31776 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 993/999 | lr: 1.34e-08 | global_batch_size: 32 | global_step: 993 | reduced_train_loss: 1.155 | train_step_timing in s: 2.39 | consumed_samples: 31808 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 994/999 | lr: 9.842e-09 | global_batch_size: 32 | global_step: 994 | reduced_train_loss: 1.114 | train_step_timing in s: 2.385 | consumed_samples: 31840 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 995/999 | lr: 6.835e-09 | global_batch_size: 32 | global_step: 995 | reduced_train_loss: 1.175 | train_step_timing in s: 2.387 | consumed_samples: 31872 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 996/999 | lr: 4.374e-09 | global_batch_size: 32 | global_step: 996 | reduced_train_loss: 1.179 | train_step_timing in s: 2.438 | consumed_samples: 31904 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 997/999 | lr: 2.461e-09 | global_batch_size: 32 | global_step: 997 | reduced_train_loss: 1.111 | train_step_timing in s: 2.384 | consumed_samples: 31936 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 998/999 | lr: 1.094e-09 | global_batch_size: 32 | global_step: 998 | reduced_train_loss: 1.025 | train_step_timing in s: 2.42 | consumed_samples: 31968 | val_loss: 1.165
s_act_ckpt/0 [default0]:Training epoch 0, iteration 999/999 | lr: 2.734e-10 | global_batch_size: 32 | global_step: 999 | reduced_train_loss: 1.225 | train_step_timing in s: 2.427 | consumed_samples: 32000 | val_loss: 1.165
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:10 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 999 : Start time: 1764083410.791s : Save duration: 0.057s
s_act_ckpt/0 [default0]:Epoch 0, global step 999: 'val_loss' was not in top 2
s_act_ckpt/0 [default0]:`Trainer.fit` stopped: `max_steps=1000` reached.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:11 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=999-consumed_samples=32000.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:11 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:11 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:11 dist_ckpt_io:190] Pending async checkpoint saves. Finalizing them synchronously now
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:11 pl:222] Successfully saved checkpoint from iteration     999 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=999-consumed_samples=32000.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:11 model_checkpoint:657] Async checkpoint save for step 1000 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_lora_4k_tokens_act_ckpt/2025-11-25_14-56-51/checkpoints/model_name=0--val_loss=1.16-step=999-consumed_samples=32000.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:11 dist_ckpt_io:160] Async finalization time took 0.407 s
s_act_ckpt/0 I1125 16:10:22.157000 1317503 torch/distributed/elastic/agent/server/api.py:888] [default] worker group successfully finished. Waiting 300 seconds for other agents to finish.
s_act_ckpt/0 I1125 16:10:22.159000 1317503 torch/distributed/elastic/agent/server/api.py:941] Local worker group finished (WorkerState.SUCCEEDED). Waiting 300 seconds for other agents to finish
s_act_ckpt/0 I1125 16:10:22.160000 1317503 torch/distributed/elastic/agent/server/api.py:955] Done waiting for other agents. Elapsed: 0.00024366378784179688 seconds
[16:10:23] INFO     Job                                          launcher.py:160
                    qwen3_600m_lora_4k_tokens_act_ckpt-xzg17zc4z                
                    tp6mc finished: SUCCEEDED                                   
                                                                                
# The experiment was run with the following tasks: ['qwen3_600m_lora_4k_tokens_a
# You can inspect and reconstruct this experiment at a later point in time using
experiment = run.Experiment.from_id("qwen3_600m_lora_4k_tokens_act_ckpt_17640789
experiment.status() # Gets the overall status                                   
experiment.logs("qwen3_600m_lora_4k_tokens_act_ckpt") # Gets the log for the pro
experiment.cancel("qwen3_600m_lora_4k_tokens_act_ckpt") # Cancels the provided t
                                                                                
                                                                                
# You can inspect this experiment at a later point in time using the CLI as well
nemo experiment status qwen3_600m_lora_4k_tokens_act_ckpt_1764078984            
nemo experiment logs qwen3_600m_lora_4k_tokens_act_ckpt_1764078984 0            
nemo experiment cancel qwen3_600m_lora_4k_tokens_act_ckpt_1764078984 0