[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/nemo/qwen3_600m_finetune_34702.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

python "$1"/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
─ Entering Experiment qwen3_600m_full_4k_tokens_act_ckpt with id: qwen3_600m_… ─
[15:40:43] INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_full_4k_tokens_act_ck                       
                    pt/qwen3_600m_full_4k_tokens_act_ckpt                       
                    _1764081643/qwen3_600m_full_4k_tokens                       
                    _act_ckpt                                                   
[15:40:43] Launching job qwen3_600m_full_4k_tokens_act_ckpt    experiment.py:744
           for experiment qwen3_600m_full_4k_tokens_act_ckpt                    
           INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_full_4k_tokens_act_ck                       
                    pt/qwen3_600m_full_4k_tokens_act_ckpt                       
                    _1764081643/qwen3_600m_full_4k_tokens                       
                    _act_ckpt                                                   
           INFO     Launched app:                                launcher.py:111
                    local_persistent://nemo_run/qwen3_600m_full_                
                    4k_tokens_act_ckpt-cnml3ktmmbsvl                            
─ Waiting for Experiment qwen3_600m_full_4k_tokens_act_ckpt_1764081643 to fin… ─

Experiment Status for qwen3_600m_full_4k_tokens_act_ckpt_1764081643

Task 0: qwen3_600m_full_4k_tokens_act_ckpt
- Status: RUNNING
- Executor: LocalExecutor
- Job id: qwen3_600m_full_4k_tokens_act_ckpt-cnml3ktmmbsvl
- Local Directory: /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_full_4k_tokens_act_ckpt/qwen3_600m_full_4k_tokens_act_ckpt_1764081643/qwen3_600m_full_4k_tokens_act_ckpt

           INFO     Waiting for job                              launcher.py:130
                    qwen3_600m_full_4k_tokens_act_ckpt-cnml3ktmm                
                    bsvl to finish [log=True]...                                
s_act_ckpt/0 /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
s_act_ckpt/0   import pynvml  # type: ignore[import]
s_act_ckpt/0 I1125 15:40:46.174000 792151 torch/distributed/run.py:657] Using nproc_per_node=1.
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199] Starting elastic_operator with launch configs:
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   entrypoint         : nemo_run.core.runners.fdl_runner
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   min_nodes          : 1
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   max_nodes          : 1
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   nproc_per_node     : 1
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   run_id             : 58
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   rdzv_backend       : c10d
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   rdzv_endpoint      : localhost:0
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   rdzv_configs       : {'timeout': 900}
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   max_restarts       : 0
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   monitor_interval   : 0.1
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   log_dir            : /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_full_4k_tokens_act_ckpt/qwen3_600m_full_4k_tokens_act_ckpt_1764081643/qwen3_600m_full_4k_tokens_act_ckpt/nemo_run/qwen3_600m_full_4k_tokens_act_ckpt-cnml3ktmmbsvl/torchelastic/qwen3_600m_full_4k_tokens_act_ckpt
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   metrics_cfg        : {}
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199]   event_log_handler  : null
s_act_ckpt/0 I1125 15:40:46.178000 792151 torch/distributed/launcher/api.py:199] 
s_act_ckpt/0 I1125 15:40:46.184000 792151 torch/distributed/elastic/agent/server/api.py:869] [default] starting workers for entrypoint: python3
s_act_ckpt/0 I1125 15:40:46.185000 792151 torch/distributed/elastic/agent/server/api.py:677] [default] Rendezvous'ing worker group
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523] [default] Rendezvous complete for workers. Result:
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   restart_count=0
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   master_addr=hai004.haicore.berlin
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   master_port=44871
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   group_rank=0
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   group_world_size=1
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   local_ranks=[0]
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   role_ranks=[0]
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   global_ranks=[0]
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   role_world_sizes=[1]
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   global_world_sizes=[1]
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523]   event_log_handler=null
s_act_ckpt/0 I1125 15:40:46.258000 792151 torch/distributed/elastic/agent/server/api.py:523] 
s_act_ckpt/0 I1125 15:40:46.259000 792151 torch/distributed/elastic/agent/server/api.py:685] [default] Starting worker group
s_act_ckpt/0 I1125 15:40:46.260000 792151 torch/distributed/elastic/agent/server/local_elastic_agent.py:296] use_agent_store: True
s_act_ckpt/0 I1125 15:40:46.260000 792151 torch/distributed/elastic/agent/server/local_elastic_agent.py:192] Environment variable 'TORCHELASTIC_ENABLE_FILE_TIMER' not found. Do not start FileTimerServer.
s_act_ckpt/0 I1125 15:40:46.261000 792151 torch/distributed/elastic/agent/server/local_elastic_agent.py:236] Environment variable 'TORCHELASTIC_HEALTH_CHECK_PORT' not found. Do not start health check.
s_act_ckpt/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
s_act_ckpt/0 [default0]:  import pynvml  # type: ignore[import]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:07 nemo_logger:162] Experiments will be logged at /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07
s_act_ckpt/0 [default0]:GPU available: True (cuda), used: True
s_act_ckpt/0 [default0]:TPU available: False, using: 0 TPU cores
s_act_ckpt/0 [default0]:HPU available: False, using: 0 HPUs
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 15:41:07 nemo_logger:190] "update_logger_directory" is True. Overwriting tensorboard logger "save_dir" to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/tb_logs
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 15:41:07 nemo_logger:209] The Trainer already contains a ModelCheckpoint callback. This will be overwritten.
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 15:41:07 nemo_logger:239] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_strategy:460] Fixing mis-match between ddp-config & mcore-optimizer config
s_act_ckpt/0 [default0]:You are using a CUDA device ('NVIDIA H100 80GB HBM3') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:449] Rank 0 has data parallel group : [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:455] Rank 0 has combined group of data parallel and context parallel : [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:460] All data parallel group ranks with context parallel combined: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:463] Ranks 0 has data parallel rank: 0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:471] Rank 0 has context parallel group: [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:474] All context parallel group ranks: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:475] Ranks 0 has context parallel rank: 0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:482] Rank 0 has model parallel group: [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:483] All model parallel group ranks: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:492] Rank 0 has tensor model parallel group: [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:496] All tensor model parallel group ranks: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:497] Rank 0 has tensor model parallel rank: 0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:532] Rank 0 has pipeline model parallel group: [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:544] Rank 0 has embedding group: [0]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:550] All pipeline model parallel group ranks: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:551] Rank 0 has pipeline model parallel rank 0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:552] All embedding group ranks: [[0]]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:08 megatron_init:553] Rank 0 has embedding rank: 0
s_act_ckpt/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
s_act_ckpt/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
s_act_ckpt/0 [default0]:[W1125 15:41:08.199398603 ProcessGroupNCCL.cpp:981] Warning: TORCH_NCCL_AVOID_RECORD_STREAMS is the default now, this environment variable is thus deprecated. (function operator())
s_act_ckpt/0 [default0]:----------------------------------------------------------------------------------------------------
s_act_ckpt/0 [default0]:distributed_backend=nccl
s_act_ckpt/0 [default0]:All distributed processes registered. Starting with 1 processes
s_act_ckpt/0 [default0]:----------------------------------------------------------------------------------------------------
s_act_ckpt/0 [default0]:
s_act_ckpt/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
s_act_ckpt/0 [default0]:[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 base:373] Use preset vocab_size: 151936, original vocab_size: 151669, dummy tokens: 267.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 megatron_strategy:472] Copying Trainer's 'max_steps' (1000) to LR scheduler's 'max_steps'.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 num_microbatches_calculator:228] setting number of microbatches to constant 8
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 megatron_parallel:584]  > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 596049920
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 utils:673] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_megatron_fsdp=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache=False, nccl_ub=False, fsdp_double_buffer=False, outer_dp_sharding_strategy='no_shard', disable_symmetric_registration=False, delay_wgrad_compute=False)
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 utils:694] Number of buckets for gradient all-reduce / reduce-scatter: 1
s_act_ckpt/0 [default0]:    Params for bucket 1 (596049920 elements, 596049920 padded size):
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.embedding.word_embeddings.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.final_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.4.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.17.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 15:41:09 serialization:184] DEPRECATED: Passing 'checkpoint_dir' as a Path object in load_common_state_dict will no longer be supported in a future release. Please pass it as a string instead.
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.13.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.22.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.20.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.8.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.7.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.27.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.10.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.11.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.self_attention.q_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.18.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.26.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.15.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.9.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.3.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.25.self_attention.linear_qkv.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.16.mlp.linear_fc2.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.2.mlp.linear_fc1.layer_norm_weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.mlp.linear_fc1.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.1.self_attention.k_layernorm.weight
s_act_ckpt/0 [default0]:    	module.decoder.layers.0.self_attention.linear_proj.weight
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 utils:673] Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=5e-06, min_lr=None, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='delayed', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.98, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather=False, overlap_param_gather_with_optimizer_step=False, use_megatron_fsdp=False, optimizer_cpu_offload=False, optimizer_offload_fraction=0.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=False, timers=None, config_logger_dir='')
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 megatron_strategy:1045] Doing selective restore from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 pl:411] Loaded sharded_state_dict_metadata from checkpoint: {'distrib_optim_sharding_type': 'fully_sharded_model_space'}
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 pl:275] Using <megatron.core.dist_checkpointing.strategies.fully_parallel.FullyParallelLoadStrategyWrapper object at 0x7f59e8100340> dist-ckpt load strategy.
s_act_ckpt/0 [default0]:
s_act_ckpt/0 [default0]:  | Name   | Type | Params | Mode 
s_act_ckpt/0 [default0]:----------------------------------------
s_act_ckpt/0 [default0]:0 | module | DDP  | 596 M  | train
s_act_ckpt/0 [default0]:----------------------------------------
s_act_ckpt/0 [default0]:596 M     Trainable params
s_act_ckpt/0 [default0]:0         Non-trainable params
s_act_ckpt/0 [default0]:596 M     Total params
s_act_ckpt/0 [default0]:2,384.200 Total estimated model params size (MB)
s_act_ckpt/0 [default0]:571       Modules in train mode
s_act_ckpt/0 [default0]:0         Modules in eval mode
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 pl:297] Global Checkpoint Load : Rank : 0 : Start time : 1764081669.389s : Time spent in load_checkpoint: 0.582s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 megatron_strategy:1050] Restoring model weights from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True)
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:41:09 megatron_strategy:1058] Finished restoring from RestoreConfig(path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m', load_model_state=True, load_optim_state=False, load_artifacts=True), cleaning up.
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 15:41:29 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('global_batch_size', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 15:41:29 nemo_logging:361] /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:431: It is recommended to use `self.log('val_loss', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
s_act_ckpt/0 [default0]:    
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 15:41:49 rerun_state_machine:1300] Implicit initialization of Rerun State Machine!
s_act_ckpt/0 [default0]:[NeMo W 2025-11-25 15:41:49 rerun_state_machine:238] RerunStateMachine initialized in mode disabled
s_act_ckpt/0 [default0]:Training epoch 0, iteration 0/999 | lr: 9.804e-08 | global_batch_size: 32 | global_step: 0 | reduced_train_loss: 1.102 | train_step_timing in s: 18.86
s_act_ckpt/0 [default0]:Training epoch 0, iteration 1/999 | lr: 1.961e-07 | global_batch_size: 32 | global_step: 1 | reduced_train_loss: 1.09 | train_step_timing in s: 2.909 | consumed_samples: 64
s_act_ckpt/0 [default0]:Training epoch 0, iteration 2/999 | lr: 2.941e-07 | global_batch_size: 32 | global_step: 2 | reduced_train_loss: 1.031 | train_step_timing in s: 2.904 | consumed_samples: 96
s_act_ckpt/0 [default0]:Training epoch 0, iteration 3/999 | lr: 3.922e-07 | global_batch_size: 32 | global_step: 3 | reduced_train_loss: 1.285 | train_step_timing in s: 2.895 | consumed_samples: 128
s_act_ckpt/0 [default0]:Training epoch 0, iteration 4/999 | lr: 4.902e-07 | global_batch_size: 32 | global_step: 4 | reduced_train_loss: 1.217 | train_step_timing in s: 2.897 | consumed_samples: 160
s_act_ckpt/0 [default0]:Training epoch 0, iteration 5/999 | lr: 5.882e-07 | global_batch_size: 32 | global_step: 5 | reduced_train_loss: 0.9809 | train_step_timing in s: 2.904 | consumed_samples: 192
s_act_ckpt/0 [default0]:Training epoch 0, iteration 6/999 | lr: 6.863e-07 | global_batch_size: 32 | global_step: 6 | reduced_train_loss: 0.9845 | train_step_timing in s: 2.904 | consumed_samples: 224
s_act_ckpt/0 [default0]:Training epoch 0, iteration 7/999 | lr: 7.843e-07 | global_batch_size: 32 | global_step: 7 | reduced_train_loss: 1.199 | train_step_timing in s: 2.897 | consumed_samples: 256
s_act_ckpt/0 [default0]:Training epoch 0, iteration 8/999 | lr: 8.824e-07 | global_batch_size: 32 | global_step: 8 | reduced_train_loss: 1.129 | train_step_timing in s: 2.909 | consumed_samples: 288
s_act_ckpt/0 [default0]:Training epoch 0, iteration 9/999 | lr: 9.804e-07 | global_batch_size: 32 | global_step: 9 | reduced_train_loss: 1.082 | train_step_timing in s: 2.905 | consumed_samples: 320
s_act_ckpt/0 [default0]:Training epoch 0, iteration 10/999 | lr: 1.078e-06 | global_batch_size: 32 | global_step: 10 | reduced_train_loss: 1.105 | train_step_timing in s: 2.904 | consumed_samples: 352
s_act_ckpt/0 [default0]:Training epoch 0, iteration 11/999 | lr: 1.176e-06 | global_batch_size: 32 | global_step: 11 | reduced_train_loss: 0.9158 | train_step_timing in s: 2.912 | consumed_samples: 384
s_act_ckpt/0 [default0]:Training epoch 0, iteration 12/999 | lr: 1.275e-06 | global_batch_size: 32 | global_step: 12 | reduced_train_loss: 0.8855 | train_step_timing in s: 2.913 | consumed_samples: 416
s_act_ckpt/0 [default0]:Training epoch 0, iteration 13/999 | lr: 1.373e-06 | global_batch_size: 32 | global_step: 13 | reduced_train_loss: 0.9916 | train_step_timing in s: 2.908 | consumed_samples: 448
s_act_ckpt/0 [default0]:Training epoch 0, iteration 14/999 | lr: 1.471e-06 | global_batch_size: 32 | global_step: 14 | reduced_train_loss: 0.9859 | train_step_timing in s: 2.914 | consumed_samples: 480
s_act_ckpt/0 [default0]:Training epoch 0, iteration 15/999 | lr: 1.569e-06 | global_batch_size: 32 | global_step: 15 | reduced_train_loss: 0.9037 | train_step_timing in s: 2.91 | consumed_samples: 512
s_act_ckpt/0 [default0]:Training epoch 0, iteration 16/999 | lr: 1.667e-06 | global_batch_size: 32 | global_step: 16 | reduced_train_loss: 0.9466 | train_step_timing in s: 2.922 | consumed_samples: 544
s_act_ckpt/0 [default0]:Training epoch 0, iteration 17/999 | lr: 1.765e-06 | global_batch_size: 32 | global_step: 17 | reduced_train_loss: 0.8624 | train_step_timing in s: 2.919 | consumed_samples: 576
s_act_ckpt/0 [default0]:Training epoch 0, iteration 18/999 | lr: 1.863e-06 | global_batch_size: 32 | global_step: 18 | reduced_train_loss: 0.7897 | train_step_timing in s: 2.911 | consumed_samples: 608
s_act_ckpt/0 [default0]:Training epoch 0, iteration 19/999 | lr: 1.961e-06 | global_batch_size: 32 | global_step: 19 | reduced_train_loss: 0.7637 | train_step_timing in s: 2.915 | consumed_samples: 640
s_act_ckpt/0 [default0]:Training epoch 0, iteration 20/999 | lr: 2.059e-06 | global_batch_size: 32 | global_step: 20 | reduced_train_loss: 1.051 | train_step_timing in s: 2.908 | consumed_samples: 672
s_act_ckpt/0 [default0]:Training epoch 0, iteration 21/999 | lr: 2.157e-06 | global_batch_size: 32 | global_step: 21 | reduced_train_loss: 0.7579 | train_step_timing in s: 2.914 | consumed_samples: 704
s_act_ckpt/0 [default0]:Training epoch 0, iteration 22/999 | lr: 2.255e-06 | global_batch_size: 32 | global_step: 22 | reduced_train_loss: 0.8357 | train_step_timing in s: 2.919 | consumed_samples: 736
s_act_ckpt/0 [default0]:Training epoch 0, iteration 23/999 | lr: 2.353e-06 | global_batch_size: 32 | global_step: 23 | reduced_train_loss: 0.7989 | train_step_timing in s: 2.922 | consumed_samples: 768
s_act_ckpt/0 [default0]:Training epoch 0, iteration 24/999 | lr: 2.451e-06 | global_batch_size: 32 | global_step: 24 | reduced_train_loss: 0.9274 | train_step_timing in s: 2.912 | consumed_samples: 800
s_act_ckpt/0 [default0]:Training epoch 0, iteration 25/999 | lr: 2.549e-06 | global_batch_size: 32 | global_step: 25 | reduced_train_loss: 0.7166 | train_step_timing in s: 2.911 | consumed_samples: 832
s_act_ckpt/0 [default0]:Training epoch 0, iteration 26/999 | lr: 2.647e-06 | global_batch_size: 32 | global_step: 26 | reduced_train_loss: 0.7541 | train_step_timing in s: 2.918 | consumed_samples: 864
s_act_ckpt/0 [default0]:Training epoch 0, iteration 27/999 | lr: 2.745e-06 | global_batch_size: 32 | global_step: 27 | reduced_train_loss: 0.6156 | train_step_timing in s: 2.915 | consumed_samples: 896
s_act_ckpt/0 [default0]:Training epoch 0, iteration 28/999 | lr: 2.843e-06 | global_batch_size: 32 | global_step: 28 | reduced_train_loss: 0.6855 | train_step_timing in s: 2.916 | consumed_samples: 928
s_act_ckpt/0 [default0]:Training epoch 0, iteration 29/999 | lr: 2.941e-06 | global_batch_size: 32 | global_step: 29 | reduced_train_loss: 0.6927 | train_step_timing in s: 2.918 | consumed_samples: 960
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:41 pl:353] Using FullyParallelSaveStrategyWrapper(torch_dist, 1) dist-ckpt save strategy.
s_act_ckpt/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
s_act_ckpt/0 [default0]:  import pynvml  # type: ignore[import]
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:46 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 30 : Start time: 1764081821.102s : Save duration: 5.475s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:49 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.69-step=29-consumed_samples=960.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 30/999 | lr: 3.039e-06 | global_batch_size: 32 | global_step: 30 | reduced_train_loss: 0.5359 | train_step_timing in s: 2.907 | consumed_samples: 992 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:52 dist_ckpt_io:160] Async finalization time took 0.001 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 31/999 | lr: 3.137e-06 | global_batch_size: 32 | global_step: 31 | reduced_train_loss: 0.7227 | train_step_timing in s: 2.896 | consumed_samples: 1024 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:55 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 32/999 | lr: 3.235e-06 | global_batch_size: 32 | global_step: 32 | reduced_train_loss: 0.7668 | train_step_timing in s: 2.9 | consumed_samples: 1056 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:43:58 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 33/999 | lr: 3.333e-06 | global_batch_size: 32 | global_step: 33 | reduced_train_loss: 0.5318 | train_step_timing in s: 2.905 | consumed_samples: 1088 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:01 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 34/999 | lr: 3.431e-06 | global_batch_size: 32 | global_step: 34 | reduced_train_loss: 0.7819 | train_step_timing in s: 2.904 | consumed_samples: 1120 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:04 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 35/999 | lr: 3.529e-06 | global_batch_size: 32 | global_step: 35 | reduced_train_loss: 0.4479 | train_step_timing in s: 2.914 | consumed_samples: 1152 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:07 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 36/999 | lr: 3.627e-06 | global_batch_size: 32 | global_step: 36 | reduced_train_loss: 0.4578 | train_step_timing in s: 2.905 | consumed_samples: 1184 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:10 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 37/999 | lr: 3.725e-06 | global_batch_size: 32 | global_step: 37 | reduced_train_loss: 0.6165 | train_step_timing in s: 2.906 | consumed_samples: 1216 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:13 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 38/999 | lr: 3.824e-06 | global_batch_size: 32 | global_step: 38 | reduced_train_loss: 0.7277 | train_step_timing in s: 2.909 | consumed_samples: 1248 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:16 pl:222] Successfully saved checkpoint from iteration      30 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.69-step=29-consumed_samples=960.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:16 model_checkpoint:657] Async checkpoint save for step 30 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.69-step=29-consumed_samples=960.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:16 dist_ckpt_io:160] Async finalization time took 0.055 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 39/999 | lr: 3.922e-06 | global_batch_size: 32 | global_step: 39 | reduced_train_loss: 0.5618 | train_step_timing in s: 2.906 | consumed_samples: 1280 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 40/999 | lr: 4.02e-06 | global_batch_size: 32 | global_step: 40 | reduced_train_loss: 0.5321 | train_step_timing in s: 2.909 | consumed_samples: 1312 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 41/999 | lr: 4.118e-06 | global_batch_size: 32 | global_step: 41 | reduced_train_loss: 0.584 | train_step_timing in s: 2.912 | consumed_samples: 1344 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 42/999 | lr: 4.216e-06 | global_batch_size: 32 | global_step: 42 | reduced_train_loss: 0.4652 | train_step_timing in s: 2.916 | consumed_samples: 1376 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 43/999 | lr: 4.314e-06 | global_batch_size: 32 | global_step: 43 | reduced_train_loss: 0.5073 | train_step_timing in s: 2.915 | consumed_samples: 1408 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 44/999 | lr: 4.412e-06 | global_batch_size: 32 | global_step: 44 | reduced_train_loss: 0.5375 | train_step_timing in s: 2.921 | consumed_samples: 1440 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 45/999 | lr: 4.51e-06 | global_batch_size: 32 | global_step: 45 | reduced_train_loss: 0.4751 | train_step_timing in s: 2.91 | consumed_samples: 1472 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 46/999 | lr: 4.608e-06 | global_batch_size: 32 | global_step: 46 | reduced_train_loss: 0.3951 | train_step_timing in s: 2.911 | consumed_samples: 1504 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 47/999 | lr: 4.706e-06 | global_batch_size: 32 | global_step: 47 | reduced_train_loss: 0.5243 | train_step_timing in s: 2.909 | consumed_samples: 1536 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 48/999 | lr: 4.804e-06 | global_batch_size: 32 | global_step: 48 | reduced_train_loss: 0.4763 | train_step_timing in s: 2.906 | consumed_samples: 1568 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 49/999 | lr: 4.902e-06 | global_batch_size: 32 | global_step: 49 | reduced_train_loss: 0.5388 | train_step_timing in s: 2.905 | consumed_samples: 1600 | val_loss: 0.687
s_act_ckpt/0 [default0]:Epoch 0, global step 49: 'val_loss' reached 0.68699 (best 0.68699), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.69-step=49-consumed_samples=1600.0.ckpt' as top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:48 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 49 : Start time: 1764081888.454s : Save duration: 0.188s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:49 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.69-step=49-consumed_samples=1600.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:49 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 50/999 | lr: 5e-06 | global_batch_size: 32 | global_step: 50 | reduced_train_loss: 0.4798 | train_step_timing in s: 2.909 | consumed_samples: 1632 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:52 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 51/999 | lr: 5e-06 | global_batch_size: 32 | global_step: 51 | reduced_train_loss: 0.5711 | train_step_timing in s: 2.91 | consumed_samples: 1664 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:55 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 52/999 | lr: 5e-06 | global_batch_size: 32 | global_step: 52 | reduced_train_loss: 0.4146 | train_step_timing in s: 2.912 | consumed_samples: 1696 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:44:57 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 53/999 | lr: 5e-06 | global_batch_size: 32 | global_step: 53 | reduced_train_loss: 0.5136 | train_step_timing in s: 2.91 | consumed_samples: 1728 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:00 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 54/999 | lr: 5e-06 | global_batch_size: 32 | global_step: 54 | reduced_train_loss: 0.403 | train_step_timing in s: 2.908 | consumed_samples: 1760 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:03 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 55/999 | lr: 5e-06 | global_batch_size: 32 | global_step: 55 | reduced_train_loss: 0.493 | train_step_timing in s: 2.908 | consumed_samples: 1792 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:06 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 56/999 | lr: 5e-06 | global_batch_size: 32 | global_step: 56 | reduced_train_loss: 0.6054 | train_step_timing in s: 2.923 | consumed_samples: 1824 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:09 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 57/999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 57 | reduced_train_loss: 0.479 | train_step_timing in s: 2.913 | consumed_samples: 1856 | val_loss: 0.687
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:12 pl:222] Successfully saved checkpoint from iteration      49 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.69-step=49-consumed_samples=1600.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:12 model_checkpoint:657] Async checkpoint save for step 50 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.69-step=49-consumed_samples=1600.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:13 dist_ckpt_io:160] Async finalization time took 0.644 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 58/999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 58 | reduced_train_loss: 0.4639 | train_step_timing in s: 2.91 | consumed_samples: 1888 | val_loss: 0.687
s_act_ckpt/0 [default0]:Training epoch 0, iteration 59/999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 59 | reduced_train_loss: 0.5128 | train_step_timing in s: 2.912 | consumed_samples: 1920 | val_loss: 0.687
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:46 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 60 : Start time: 1764081946.118s : Save duration: 0.182s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:46 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.46-step=59-consumed_samples=1920.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 60/999 | lr: 4.999e-06 | global_batch_size: 32 | global_step: 60 | reduced_train_loss: 0.5704 | train_step_timing in s: 2.909 | consumed_samples: 1952 | val_loss: 0.4561
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:49 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 61/999 | lr: 4.998e-06 | global_batch_size: 32 | global_step: 61 | reduced_train_loss: 0.517 | train_step_timing in s: 2.906 | consumed_samples: 1984 | val_loss: 0.4561
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:52 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 62/999 | lr: 4.998e-06 | global_batch_size: 32 | global_step: 62 | reduced_train_loss: 0.4565 | train_step_timing in s: 2.91 | consumed_samples: 2016 | val_loss: 0.4561
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:55 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 63/999 | lr: 4.998e-06 | global_batch_size: 32 | global_step: 63 | reduced_train_loss: 0.4878 | train_step_timing in s: 2.907 | consumed_samples: 2048 | val_loss: 0.4561
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:45:58 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 64/999 | lr: 4.997e-06 | global_batch_size: 32 | global_step: 64 | reduced_train_loss: 0.4434 | train_step_timing in s: 2.906 | consumed_samples: 2080 | val_loss: 0.4561
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:46:01 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 65/999 | lr: 4.997e-06 | global_batch_size: 32 | global_step: 65 | reduced_train_loss: 0.4842 | train_step_timing in s: 2.912 | consumed_samples: 2112 | val_loss: 0.4561
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:46:04 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 66/999 | lr: 4.997e-06 | global_batch_size: 32 | global_step: 66 | reduced_train_loss: 0.4894 | train_step_timing in s: 2.907 | consumed_samples: 2144 | val_loss: 0.4561
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:46:07 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 67/999 | lr: 4.996e-06 | global_batch_size: 32 | global_step: 67 | reduced_train_loss: 0.4954 | train_step_timing in s: 2.906 | consumed_samples: 2176 | val_loss: 0.4561
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:46:10 pl:222] Successfully saved checkpoint from iteration      60 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.46-step=59-consumed_samples=1920.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:46:10 model_checkpoint:657] Async checkpoint save for step 60 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.46-step=59-consumed_samples=1920.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:46:10 dist_ckpt_io:160] Async finalization time took 0.060 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 68/999 | lr: 4.996e-06 | global_batch_size: 32 | global_step: 68 | reduced_train_loss: 0.461 | train_step_timing in s: 2.908 | consumed_samples: 2208 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 69/999 | lr: 4.995e-06 | global_batch_size: 32 | global_step: 69 | reduced_train_loss: 0.3713 | train_step_timing in s: 2.904 | consumed_samples: 2240 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 70/999 | lr: 4.995e-06 | global_batch_size: 32 | global_step: 70 | reduced_train_loss: 0.47 | train_step_timing in s: 2.914 | consumed_samples: 2272 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 71/999 | lr: 4.994e-06 | global_batch_size: 32 | global_step: 71 | reduced_train_loss: 0.3699 | train_step_timing in s: 2.903 | consumed_samples: 2304 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 72/999 | lr: 4.993e-06 | global_batch_size: 32 | global_step: 72 | reduced_train_loss: 0.4645 | train_step_timing in s: 2.909 | consumed_samples: 2336 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 73/999 | lr: 4.993e-06 | global_batch_size: 32 | global_step: 73 | reduced_train_loss: 0.4517 | train_step_timing in s: 2.914 | consumed_samples: 2368 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 74/999 | lr: 4.992e-06 | global_batch_size: 32 | global_step: 74 | reduced_train_loss: 0.3669 | train_step_timing in s: 2.918 | consumed_samples: 2400 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 75/999 | lr: 4.991e-06 | global_batch_size: 32 | global_step: 75 | reduced_train_loss: 0.621 | train_step_timing in s: 2.916 | consumed_samples: 2432 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 76/999 | lr: 4.991e-06 | global_batch_size: 32 | global_step: 76 | reduced_train_loss: 0.4645 | train_step_timing in s: 2.908 | consumed_samples: 2464 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 77/999 | lr: 4.99e-06 | global_batch_size: 32 | global_step: 77 | reduced_train_loss: 0.3668 | train_step_timing in s: 2.913 | consumed_samples: 2496 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 78/999 | lr: 4.989e-06 | global_batch_size: 32 | global_step: 78 | reduced_train_loss: 0.3997 | train_step_timing in s: 2.91 | consumed_samples: 2528 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 79/999 | lr: 4.989e-06 | global_batch_size: 32 | global_step: 79 | reduced_train_loss: 0.6763 | train_step_timing in s: 2.912 | consumed_samples: 2560 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 80/999 | lr: 4.988e-06 | global_batch_size: 32 | global_step: 80 | reduced_train_loss: 0.3763 | train_step_timing in s: 2.909 | consumed_samples: 2592 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 81/999 | lr: 4.987e-06 | global_batch_size: 32 | global_step: 81 | reduced_train_loss: 0.5452 | train_step_timing in s: 2.914 | consumed_samples: 2624 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 82/999 | lr: 4.986e-06 | global_batch_size: 32 | global_step: 82 | reduced_train_loss: 0.3475 | train_step_timing in s: 2.913 | consumed_samples: 2656 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 83/999 | lr: 4.985e-06 | global_batch_size: 32 | global_step: 83 | reduced_train_loss: 0.4152 | train_step_timing in s: 2.902 | consumed_samples: 2688 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 84/999 | lr: 4.984e-06 | global_batch_size: 32 | global_step: 84 | reduced_train_loss: 0.3388 | train_step_timing in s: 2.906 | consumed_samples: 2720 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 85/999 | lr: 4.983e-06 | global_batch_size: 32 | global_step: 85 | reduced_train_loss: 0.3786 | train_step_timing in s: 2.902 | consumed_samples: 2752 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 86/999 | lr: 4.982e-06 | global_batch_size: 32 | global_step: 86 | reduced_train_loss: 0.41 | train_step_timing in s: 2.91 | consumed_samples: 2784 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 87/999 | lr: 4.981e-06 | global_batch_size: 32 | global_step: 87 | reduced_train_loss: 0.3929 | train_step_timing in s: 2.913 | consumed_samples: 2816 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 88/999 | lr: 4.98e-06 | global_batch_size: 32 | global_step: 88 | reduced_train_loss: 0.3255 | train_step_timing in s: 2.901 | consumed_samples: 2848 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Training epoch 0, iteration 89/999 | lr: 4.979e-06 | global_batch_size: 32 | global_step: 89 | reduced_train_loss: 0.3956 | train_step_timing in s: 2.91 | consumed_samples: 2880 | val_loss: 0.4561
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:42 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 90 : Start time: 1764082061.972s : Save duration: 0.188s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:42 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.36-step=89-consumed_samples=2880.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 90/999 | lr: 4.978e-06 | global_batch_size: 32 | global_step: 90 | reduced_train_loss: 0.3805 | train_step_timing in s: 2.904 | consumed_samples: 2912 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:45 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 91/999 | lr: 4.977e-06 | global_batch_size: 32 | global_step: 91 | reduced_train_loss: 0.3544 | train_step_timing in s: 2.896 | consumed_samples: 2944 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:48 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 92/999 | lr: 4.976e-06 | global_batch_size: 32 | global_step: 92 | reduced_train_loss: 0.5176 | train_step_timing in s: 2.904 | consumed_samples: 2976 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:51 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 93/999 | lr: 4.975e-06 | global_batch_size: 32 | global_step: 93 | reduced_train_loss: 0.3434 | train_step_timing in s: 2.907 | consumed_samples: 3008 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:54 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 94/999 | lr: 4.974e-06 | global_batch_size: 32 | global_step: 94 | reduced_train_loss: 0.3921 | train_step_timing in s: 2.904 | consumed_samples: 3040 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:47:57 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 95/999 | lr: 4.972e-06 | global_batch_size: 32 | global_step: 95 | reduced_train_loss: 0.3645 | train_step_timing in s: 2.909 | consumed_samples: 3072 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:00 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 96/999 | lr: 4.971e-06 | global_batch_size: 32 | global_step: 96 | reduced_train_loss: 0.3182 | train_step_timing in s: 2.904 | consumed_samples: 3104 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:03 pl:222] Successfully saved checkpoint from iteration      90 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.36-step=89-consumed_samples=2880.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:03 model_checkpoint:657] Async checkpoint save for step 90 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.36-step=89-consumed_samples=2880.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:03 dist_ckpt_io:160] Async finalization time took 0.756 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 97/999 | lr: 4.97e-06 | global_batch_size: 32 | global_step: 97 | reduced_train_loss: 0.3572 | train_step_timing in s: 2.897 | consumed_samples: 3136 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 98/999 | lr: 4.969e-06 | global_batch_size: 32 | global_step: 98 | reduced_train_loss: 0.395 | train_step_timing in s: 2.904 | consumed_samples: 3168 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 99/999 | lr: 4.967e-06 | global_batch_size: 32 | global_step: 99 | reduced_train_loss: 0.3349 | train_step_timing in s: 2.908 | consumed_samples: 3200 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Epoch 0, global step 99: 'val_loss' reached 0.35634 (best 0.35634), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.36-step=99-consumed_samples=3200.0.ckpt' as top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:13 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 99 : Start time: 1764082092.791s : Save duration: 0.913s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:14 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.36-step=99-consumed_samples=3200.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:14 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 100/999 | lr: 4.966e-06 | global_batch_size: 32 | global_step: 100 | reduced_train_loss: 0.3184 | train_step_timing in s: 2.905 | consumed_samples: 3232 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:17 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 101/999 | lr: 4.965e-06 | global_batch_size: 32 | global_step: 101 | reduced_train_loss: 0.2884 | train_step_timing in s: 2.905 | consumed_samples: 3264 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:20 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 102/999 | lr: 4.963e-06 | global_batch_size: 32 | global_step: 102 | reduced_train_loss: 0.508 | train_step_timing in s: 2.9 | consumed_samples: 3296 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:23 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 103/999 | lr: 4.962e-06 | global_batch_size: 32 | global_step: 103 | reduced_train_loss: 0.2742 | train_step_timing in s: 2.907 | consumed_samples: 3328 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:26 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 104/999 | lr: 4.96e-06 | global_batch_size: 32 | global_step: 104 | reduced_train_loss: 0.4288 | train_step_timing in s: 2.908 | consumed_samples: 3360 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:28 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 105/999 | lr: 4.959e-06 | global_batch_size: 32 | global_step: 105 | reduced_train_loss: 0.3165 | train_step_timing in s: 2.91 | consumed_samples: 3392 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:31 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 106/999 | lr: 4.957e-06 | global_batch_size: 32 | global_step: 106 | reduced_train_loss: 0.3555 | train_step_timing in s: 2.906 | consumed_samples: 3424 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:34 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 107/999 | lr: 4.956e-06 | global_batch_size: 32 | global_step: 107 | reduced_train_loss: 0.3213 | train_step_timing in s: 2.906 | consumed_samples: 3456 | val_loss: 0.3563
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:37 pl:222] Successfully saved checkpoint from iteration      99 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.36-step=99-consumed_samples=3200.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:37 model_checkpoint:657] Async checkpoint save for step 100 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.36-step=99-consumed_samples=3200.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:48:38 dist_ckpt_io:160] Async finalization time took 0.641 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 108/999 | lr: 4.954e-06 | global_batch_size: 32 | global_step: 108 | reduced_train_loss: 0.4107 | train_step_timing in s: 2.896 | consumed_samples: 3488 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 109/999 | lr: 4.953e-06 | global_batch_size: 32 | global_step: 109 | reduced_train_loss: 0.3403 | train_step_timing in s: 2.902 | consumed_samples: 3520 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 110/999 | lr: 4.951e-06 | global_batch_size: 32 | global_step: 110 | reduced_train_loss: 0.235 | train_step_timing in s: 2.901 | consumed_samples: 3552 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 111/999 | lr: 4.949e-06 | global_batch_size: 32 | global_step: 111 | reduced_train_loss: 0.3823 | train_step_timing in s: 2.898 | consumed_samples: 3584 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 112/999 | lr: 4.948e-06 | global_batch_size: 32 | global_step: 112 | reduced_train_loss: 0.3006 | train_step_timing in s: 2.905 | consumed_samples: 3616 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 113/999 | lr: 4.946e-06 | global_batch_size: 32 | global_step: 113 | reduced_train_loss: 0.4409 | train_step_timing in s: 2.908 | consumed_samples: 3648 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 114/999 | lr: 4.944e-06 | global_batch_size: 32 | global_step: 114 | reduced_train_loss: 0.3024 | train_step_timing in s: 2.9 | consumed_samples: 3680 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 115/999 | lr: 4.942e-06 | global_batch_size: 32 | global_step: 115 | reduced_train_loss: 0.3193 | train_step_timing in s: 2.907 | consumed_samples: 3712 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 116/999 | lr: 4.941e-06 | global_batch_size: 32 | global_step: 116 | reduced_train_loss: 0.3993 | train_step_timing in s: 2.911 | consumed_samples: 3744 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 117/999 | lr: 4.939e-06 | global_batch_size: 32 | global_step: 117 | reduced_train_loss: 0.4399 | train_step_timing in s: 2.899 | consumed_samples: 3776 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 118/999 | lr: 4.937e-06 | global_batch_size: 32 | global_step: 118 | reduced_train_loss: 0.3707 | train_step_timing in s: 2.908 | consumed_samples: 3808 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Training epoch 0, iteration 119/999 | lr: 4.935e-06 | global_batch_size: 32 | global_step: 119 | reduced_train_loss: 0.2981 | train_step_timing in s: 2.903 | consumed_samples: 3840 | val_loss: 0.3563
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:49:40 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 120 : Start time: 1764082180.145s : Save duration: 0.187s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:49:40 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.32-step=119-consumed_samples=3840.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 120/999 | lr: 4.933e-06 | global_batch_size: 32 | global_step: 120 | reduced_train_loss: 0.3543 | train_step_timing in s: 2.913 | consumed_samples: 3872 | val_loss: 0.3185
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:49:43 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 121/999 | lr: 4.931e-06 | global_batch_size: 32 | global_step: 121 | reduced_train_loss: 0.3628 | train_step_timing in s: 2.905 | consumed_samples: 3904 | val_loss: 0.3185
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:49:46 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 122/999 | lr: 4.929e-06 | global_batch_size: 32 | global_step: 122 | reduced_train_loss: 0.3255 | train_step_timing in s: 2.902 | consumed_samples: 3936 | val_loss: 0.3185
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:49:49 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 123/999 | lr: 4.928e-06 | global_batch_size: 32 | global_step: 123 | reduced_train_loss: 0.4714 | train_step_timing in s: 2.902 | consumed_samples: 3968 | val_loss: 0.3185
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:49:52 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 124/999 | lr: 4.926e-06 | global_batch_size: 32 | global_step: 124 | reduced_train_loss: 0.3407 | train_step_timing in s: 2.906 | consumed_samples: 4000 | val_loss: 0.3185
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:49:55 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 125/999 | lr: 4.924e-06 | global_batch_size: 32 | global_step: 125 | reduced_train_loss: 0.4451 | train_step_timing in s: 2.902 | consumed_samples: 4032 | val_loss: 0.3185
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:49:58 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 126/999 | lr: 4.921e-06 | global_batch_size: 32 | global_step: 126 | reduced_train_loss: 0.3401 | train_step_timing in s: 2.904 | consumed_samples: 4064 | val_loss: 0.3185
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:50:01 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 127/999 | lr: 4.919e-06 | global_batch_size: 32 | global_step: 127 | reduced_train_loss: 0.2783 | train_step_timing in s: 2.901 | consumed_samples: 4096 | val_loss: 0.3185
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:50:04 pl:222] Successfully saved checkpoint from iteration     120 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.32-step=119-consumed_samples=3840.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:50:04 model_checkpoint:657] Async checkpoint save for step 120 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.32-step=119-consumed_samples=3840.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:50:04 dist_ckpt_io:160] Async finalization time took 0.054 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 128/999 | lr: 4.917e-06 | global_batch_size: 32 | global_step: 128 | reduced_train_loss: 0.428 | train_step_timing in s: 2.899 | consumed_samples: 4128 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 129/999 | lr: 4.915e-06 | global_batch_size: 32 | global_step: 129 | reduced_train_loss: 0.3995 | train_step_timing in s: 2.901 | consumed_samples: 4160 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 130/999 | lr: 4.913e-06 | global_batch_size: 32 | global_step: 130 | reduced_train_loss: 0.3719 | train_step_timing in s: 2.901 | consumed_samples: 4192 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 131/999 | lr: 4.911e-06 | global_batch_size: 32 | global_step: 131 | reduced_train_loss: 0.2913 | train_step_timing in s: 2.903 | consumed_samples: 4224 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 132/999 | lr: 4.909e-06 | global_batch_size: 32 | global_step: 132 | reduced_train_loss: 0.3802 | train_step_timing in s: 2.906 | consumed_samples: 4256 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 133/999 | lr: 4.906e-06 | global_batch_size: 32 | global_step: 133 | reduced_train_loss: 0.3577 | train_step_timing in s: 2.903 | consumed_samples: 4288 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 134/999 | lr: 4.904e-06 | global_batch_size: 32 | global_step: 134 | reduced_train_loss: 0.2407 | train_step_timing in s: 2.897 | consumed_samples: 4320 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 135/999 | lr: 4.902e-06 | global_batch_size: 32 | global_step: 135 | reduced_train_loss: 0.3103 | train_step_timing in s: 2.905 | consumed_samples: 4352 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 136/999 | lr: 4.9e-06 | global_batch_size: 32 | global_step: 136 | reduced_train_loss: 0.3362 | train_step_timing in s: 2.901 | consumed_samples: 4384 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 137/999 | lr: 4.897e-06 | global_batch_size: 32 | global_step: 137 | reduced_train_loss: 0.2628 | train_step_timing in s: 2.901 | consumed_samples: 4416 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 138/999 | lr: 4.895e-06 | global_batch_size: 32 | global_step: 138 | reduced_train_loss: 0.2633 | train_step_timing in s: 2.905 | consumed_samples: 4448 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 139/999 | lr: 4.893e-06 | global_batch_size: 32 | global_step: 139 | reduced_train_loss: 0.3637 | train_step_timing in s: 2.893 | consumed_samples: 4480 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 140/999 | lr: 4.89e-06 | global_batch_size: 32 | global_step: 140 | reduced_train_loss: 0.3073 | train_step_timing in s: 2.901 | consumed_samples: 4512 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 141/999 | lr: 4.888e-06 | global_batch_size: 32 | global_step: 141 | reduced_train_loss: 0.6137 | train_step_timing in s: 2.904 | consumed_samples: 4544 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 142/999 | lr: 4.885e-06 | global_batch_size: 32 | global_step: 142 | reduced_train_loss: 0.3212 | train_step_timing in s: 2.901 | consumed_samples: 4576 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 143/999 | lr: 4.883e-06 | global_batch_size: 32 | global_step: 143 | reduced_train_loss: 0.3408 | train_step_timing in s: 2.898 | consumed_samples: 4608 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 144/999 | lr: 4.88e-06 | global_batch_size: 32 | global_step: 144 | reduced_train_loss: 0.3645 | train_step_timing in s: 2.899 | consumed_samples: 4640 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 145/999 | lr: 4.878e-06 | global_batch_size: 32 | global_step: 145 | reduced_train_loss: 0.2839 | train_step_timing in s: 2.904 | consumed_samples: 4672 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 146/999 | lr: 4.875e-06 | global_batch_size: 32 | global_step: 146 | reduced_train_loss: 0.2294 | train_step_timing in s: 2.899 | consumed_samples: 4704 | val_loss: 0.3185[default0]:
s_act_ckpt/0 [default0]:Training epoch 0, iteration 147/999 | lr: 4.872e-06 | global_batch_size: 32 | global_step: 147 | reduced_train_loss: 0.3588 | train_step_timing in s: 2.903 | consumed_samples: 4736 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Training epoch 0, iteration 148/999 | lr: 4.87e-06 | global_batch_size: 32 | global_step: 148 | reduced_train_loss: 0.2664 | train_step_timing in s: 2.9 | consumed_samples: 4768 | val_loss: 0.3185
s_act_ckpt/0 [default0]:Epoch 0, global step 149: 'val_loss' reached 0.31848 (best 0.31848), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.32-step=149-consumed_samples=4800.0.ckpt' as top 2
s_act_ckpt/0 [default0]:Training epoch 0, iteration 149/999 | lr: 4.867e-06 | global_batch_size: 32 | global_step: 149 | reduced_train_loss: 0.3774 | train_step_timing in s: 2.904 | consumed_samples: 4800 | val_loss: 0.3185
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:51:08 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 149 : Start time: 1764082268.514s : Save duration: 0.177s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:51:09 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.32-step=149-consumed_samples=4800.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:51:09 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:Training epoch 0, iteration 150/999 | lr: 4.865e-06 | global_batch_size: 32 | global_step: 150 | reduced_train_loss: 0.2923 | train_step_timing in s: 2.903 | consumed_samples: 4832 | val_loss: 0.298
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:51:39 pl:222] Successfully saved checkpoint from iteration     149 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.32-step=149-consumed_samples=4800.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:51:39 model_checkpoint:657] Async checkpoint save for step 150 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.32-step=149-consumed_samples=4800.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:51:40 dist_ckpt_io:160] Async finalization time took 1.276 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 151/999 | lr: 4.862e-06 | global_batch_size: 32 | global_step: 151 | reduced_train_loss: 0.3578 | train_step_timing in s: 2.894 | consumed_samples: 4864 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 152/999 | lr: 4.859e-06 | global_batch_size: 32 | global_step: 152 | reduced_train_loss: 0.324 | train_step_timing in s: 2.899 | consumed_samples: 4896 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 153/999 | lr: 4.856e-06 | global_batch_size: 32 | global_step: 153 | reduced_train_loss: 0.3328 | train_step_timing in s: 2.898 | consumed_samples: 4928 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 154/999 | lr: 4.854e-06 | global_batch_size: 32 | global_step: 154 | reduced_train_loss: 0.5134 | train_step_timing in s: 2.899 | consumed_samples: 4960 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 155/999 | lr: 4.851e-06 | global_batch_size: 32 | global_step: 155 | reduced_train_loss: 0.3756 | train_step_timing in s: 2.903 | consumed_samples: 4992 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 156/999 | lr: 4.848e-06 | global_batch_size: 32 | global_step: 156 | reduced_train_loss: 0.2667 | train_step_timing in s: 2.9 | consumed_samples: 5024 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 157/999 | lr: 4.845e-06 | global_batch_size: 32 | global_step: 157 | reduced_train_loss: 0.304 | train_step_timing in s: 2.902 | consumed_samples: 5056 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 158/999 | lr: 4.842e-06 | global_batch_size: 32 | global_step: 158 | reduced_train_loss: 0.3595 | train_step_timing in s: 2.896 | consumed_samples: 5088 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 159/999 | lr: 4.839e-06 | global_batch_size: 32 | global_step: 159 | reduced_train_loss: 0.3437 | train_step_timing in s: 2.889 | consumed_samples: 5120 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 160/999 | lr: 4.836e-06 | global_batch_size: 32 | global_step: 160 | reduced_train_loss: 0.3175 | train_step_timing in s: 2.895 | consumed_samples: 5152 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 161/999 | lr: 4.833e-06 | global_batch_size: 32 | global_step: 161 | reduced_train_loss: 0.2961 | train_step_timing in s: 2.902 | consumed_samples: 5184 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 162/999 | lr: 4.83e-06 | global_batch_size: 32 | global_step: 162 | reduced_train_loss: 0.3005 | train_step_timing in s: 2.903 | consumed_samples: 5216 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 163/999 | lr: 4.827e-06 | global_batch_size: 32 | global_step: 163 | reduced_train_loss: 0.2833 | train_step_timing in s: 2.902 | consumed_samples: 5248 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 164/999 | lr: 4.824e-06 | global_batch_size: 32 | global_step: 164 | reduced_train_loss: 0.2393 | train_step_timing in s: 2.905 | consumed_samples: 5280 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 165/999 | lr: 4.821e-06 | global_batch_size: 32 | global_step: 165 | reduced_train_loss: 0.4503 | train_step_timing in s: 2.893 | consumed_samples: 5312 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 166/999 | lr: 4.818e-06 | global_batch_size: 32 | global_step: 166 | reduced_train_loss: 0.3169 | train_step_timing in s: 2.891 | consumed_samples: 5344 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 167/999 | lr: 4.815e-06 | global_batch_size: 32 | global_step: 167 | reduced_train_loss: 0.2599 | train_step_timing in s: 2.908 | consumed_samples: 5376 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 168/999 | lr: 4.812e-06 | global_batch_size: 32 | global_step: 168 | reduced_train_loss: 0.2436 | train_step_timing in s: 2.899 | consumed_samples: 5408 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 169/999 | lr: 4.809e-06 | global_batch_size: 32 | global_step: 169 | reduced_train_loss: 0.2929 | train_step_timing in s: 2.901 | consumed_samples: 5440 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 170/999 | lr: 4.806e-06 | global_batch_size: 32 | global_step: 170 | reduced_train_loss: 0.5055 | train_step_timing in s: 2.902 | consumed_samples: 5472 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 171/999 | lr: 4.803e-06 | global_batch_size: 32 | global_step: 171 | reduced_train_loss: 0.2796 | train_step_timing in s: 2.909 | consumed_samples: 5504 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 172/999 | lr: 4.799e-06 | global_batch_size: 32 | global_step: 172 | reduced_train_loss: 0.2919 | train_step_timing in s: 2.901 | consumed_samples: 5536 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 173/999 | lr: 4.796e-06 | global_batch_size: 32 | global_step: 173 | reduced_train_loss: 0.3768 | train_step_timing in s: 2.902 | consumed_samples: 5568 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 174/999 | lr: 4.793e-06 | global_batch_size: 32 | global_step: 174 | reduced_train_loss: 0.2645 | train_step_timing in s: 2.902 | consumed_samples: 5600 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 175/999 | lr: 4.789e-06 | global_batch_size: 32 | global_step: 175 | reduced_train_loss: 0.2979 | train_step_timing in s: 2.901 | consumed_samples: 5632 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 176/999 | lr: 4.786e-06 | global_batch_size: 32 | global_step: 176 | reduced_train_loss: 0.3415 | train_step_timing in s: 2.898 | consumed_samples: 5664 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 177/999 | lr: 4.783e-06 | global_batch_size: 32 | global_step: 177 | reduced_train_loss: 0.3674 | train_step_timing in s: 2.903 | consumed_samples: 5696 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 178/999 | lr: 4.779e-06 | global_batch_size: 32 | global_step: 178 | reduced_train_loss: 0.3519 | train_step_timing in s: 2.904 | consumed_samples: 5728 | val_loss: 0.298
s_act_ckpt/0 [default0]:Training epoch 0, iteration 179/999 | lr: 4.776e-06 | global_batch_size: 32 | global_step: 179 | reduced_train_loss: 0.308 | train_step_timing in s: 2.896 | consumed_samples: 5760 | val_loss: 0.298
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:31 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 180 : Start time: 1764082411.043s : Save duration: 0.652s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:32 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.29-step=179-consumed_samples=5760.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 180/999 | lr: 4.773e-06 | global_batch_size: 32 | global_step: 180 | reduced_train_loss: 0.3478 | train_step_timing in s: 2.905 | consumed_samples: 5792 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:35 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 181/999 | lr: 4.769e-06 | global_batch_size: 32 | global_step: 181 | reduced_train_loss: 0.3557 | train_step_timing in s: 2.9 | consumed_samples: 5824 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:38 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 182/999 | lr: 4.766e-06 | global_batch_size: 32 | global_step: 182 | reduced_train_loss: 0.2779 | train_step_timing in s: 2.891 | consumed_samples: 5856 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:41 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 183/999 | lr: 4.762e-06 | global_batch_size: 32 | global_step: 183 | reduced_train_loss: 0.4417 | train_step_timing in s: 2.902 | consumed_samples: 5888 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:43 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 184/999 | lr: 4.759e-06 | global_batch_size: 32 | global_step: 184 | reduced_train_loss: 0.2708 | train_step_timing in s: 2.896 | consumed_samples: 5920 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:46 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 185/999 | lr: 4.755e-06 | global_batch_size: 32 | global_step: 185 | reduced_train_loss: 0.3684 | train_step_timing in s: 2.899 | consumed_samples: 5952 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:49 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 186/999 | lr: 4.751e-06 | global_batch_size: 32 | global_step: 186 | reduced_train_loss: 0.3823 | train_step_timing in s: 2.895 | consumed_samples: 5984 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:52 pl:222] Successfully saved checkpoint from iteration     180 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.29-step=179-consumed_samples=5760.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:52 model_checkpoint:657] Async checkpoint save for step 180 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.29-step=179-consumed_samples=5760.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:53:52 dist_ckpt_io:160] Async finalization time took 0.053 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 187/999 | lr: 4.748e-06 | global_batch_size: 32 | global_step: 187 | reduced_train_loss: 0.3086 | train_step_timing in s: 2.899 | consumed_samples: 6016 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 188/999 | lr: 4.744e-06 | global_batch_size: 32 | global_step: 188 | reduced_train_loss: 0.4336 | train_step_timing in s: 2.902 | consumed_samples: 6048 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 189/999 | lr: 4.741e-06 | global_batch_size: 32 | global_step: 189 | reduced_train_loss: 0.5931 | train_step_timing in s: 2.899 | consumed_samples: 6080 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 190/999 | lr: 4.737e-06 | global_batch_size: 32 | global_step: 190 | reduced_train_loss: 0.3489 | train_step_timing in s: 2.895 | consumed_samples: 6112 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 191/999 | lr: 4.733e-06 | global_batch_size: 32 | global_step: 191 | reduced_train_loss: 0.3275 | train_step_timing in s: 2.898 | consumed_samples: 6144 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 192/999 | lr: 4.729e-06 | global_batch_size: 32 | global_step: 192 | reduced_train_loss: 0.5755 | train_step_timing in s: 2.898 | consumed_samples: 6176 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 193/999 | lr: 4.726e-06 | global_batch_size: 32 | global_step: 193 | reduced_train_loss: 0.3945 | train_step_timing in s: 2.895 | consumed_samples: 6208 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 194/999 | lr: 4.722e-06 | global_batch_size: 32 | global_step: 194 | reduced_train_loss: 0.2787 | train_step_timing in s: 2.904 | consumed_samples: 6240 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 195/999 | lr: 4.718e-06 | global_batch_size: 32 | global_step: 195 | reduced_train_loss: 0.2551 | train_step_timing in s: 2.904 | consumed_samples: 6272 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 196/999 | lr: 4.714e-06 | global_batch_size: 32 | global_step: 196 | reduced_train_loss: 0.2257 | train_step_timing in s: 2.9 | consumed_samples: 6304 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 197/999 | lr: 4.71e-06 | global_batch_size: 32 | global_step: 197 | reduced_train_loss: 0.3356 | train_step_timing in s: 2.905 | consumed_samples: 6336 | val_loss: 0.2871[default0]:
s_act_ckpt/0 [default0]:Training epoch 0, iteration 198/999 | lr: 4.707e-06 | global_batch_size: 32 | global_step: 198 | reduced_train_loss: 0.3388 | train_step_timing in s: 2.9 | consumed_samples: 6368 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 199/999 | lr: 4.703e-06 | global_batch_size: 32 | global_step: 199 | reduced_train_loss: 0.2595 | train_step_timing in s: 2.898 | consumed_samples: 6400 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Epoch 0, global step 199: 'val_loss' reached 0.28707 (best 0.28707), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.29-step=199-consumed_samples=6400.0.ckpt' as top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:30 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 199 : Start time: 1764082470.759s : Save duration: 0.183s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:31 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.29-step=199-consumed_samples=6400.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:31 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 200/999 | lr: 4.699e-06 | global_batch_size: 32 | global_step: 200 | reduced_train_loss: 0.2287 | train_step_timing in s: 2.899 | consumed_samples: 6432 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:34 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 201/999 | lr: 4.695e-06 | global_batch_size: 32 | global_step: 201 | reduced_train_loss: 0.1965 | train_step_timing in s: 2.894 | consumed_samples: 6464 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:37 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 202/999 | lr: 4.691e-06 | global_batch_size: 32 | global_step: 202 | reduced_train_loss: 0.3678 | train_step_timing in s: 2.893 | consumed_samples: 6496 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:40 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 203/999 | lr: 4.687e-06 | global_batch_size: 32 | global_step: 203 | reduced_train_loss: 0.2835 | train_step_timing in s: 2.907 | consumed_samples: 6528 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:43 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 204/999 | lr: 4.683e-06 | global_batch_size: 32 | global_step: 204 | reduced_train_loss: 0.2996 | train_step_timing in s: 2.89 | consumed_samples: 6560 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:46 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 205/999 | lr: 4.679e-06 | global_batch_size: 32 | global_step: 205 | reduced_train_loss: 0.2871 | train_step_timing in s: 2.897 | consumed_samples: 6592 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:48 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 206/999 | lr: 4.675e-06 | global_batch_size: 32 | global_step: 206 | reduced_train_loss: 0.3469 | train_step_timing in s: 2.891 | consumed_samples: 6624 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:51 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 207/999 | lr: 4.671e-06 | global_batch_size: 32 | global_step: 207 | reduced_train_loss: 0.2105 | train_step_timing in s: 2.9 | consumed_samples: 6656 | val_loss: 0.2871
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:54 pl:222] Successfully saved checkpoint from iteration     199 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.29-step=199-consumed_samples=6400.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:54 model_checkpoint:657] Async checkpoint save for step 200 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.29-step=199-consumed_samples=6400.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:54:56 dist_ckpt_io:160] Async finalization time took 1.389 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 208/999 | lr: 4.666e-06 | global_batch_size: 32 | global_step: 208 | reduced_train_loss: 0.2167 | train_step_timing in s: 2.893 | consumed_samples: 6688 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Training epoch 0, iteration 209/999 | lr: 4.662e-06 | global_batch_size: 32 | global_step: 209 | reduced_train_loss: 0.2968 | train_step_timing in s: 2.892 | consumed_samples: 6720 | val_loss: 0.2871
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:27 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 210 : Start time: 1764082527.701s : Save duration: 0.182s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:28 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.28-step=209-consumed_samples=6720.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 210/999 | lr: 4.658e-06 | global_batch_size: 32 | global_step: 210 | reduced_train_loss: 0.2724 | train_step_timing in s: 2.9 | consumed_samples: 6752 | val_loss: 0.279
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:31 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 211/999 | lr: 4.654e-06 | global_batch_size: 32 | global_step: 211 | reduced_train_loss: 0.2457 | train_step_timing in s: 2.896 | consumed_samples: 6784 | val_loss: 0.279
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:34 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 212/999 | lr: 4.65e-06 | global_batch_size: 32 | global_step: 212 | reduced_train_loss: 0.3677 | train_step_timing in s: 2.896 | consumed_samples: 6816 | val_loss: 0.279
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:37 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 213/999 | lr: 4.646e-06 | global_batch_size: 32 | global_step: 213 | reduced_train_loss: 0.2665 | train_step_timing in s: 2.898 | consumed_samples: 6848 | val_loss: 0.279
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:40 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 214/999 | lr: 4.641e-06 | global_batch_size: 32 | global_step: 214 | reduced_train_loss: 0.3441 | train_step_timing in s: 2.89 | consumed_samples: 6880 | val_loss: 0.279
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:42 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 215/999 | lr: 4.637e-06 | global_batch_size: 32 | global_step: 215 | reduced_train_loss: 0.2691 | train_step_timing in s: 2.888 | consumed_samples: 6912 | val_loss: 0.279
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:45 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 216/999 | lr: 4.633e-06 | global_batch_size: 32 | global_step: 216 | reduced_train_loss: 0.3965 | train_step_timing in s: 2.899 | consumed_samples: 6944 | val_loss: 0.279
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:48 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 217/999 | lr: 4.628e-06 | global_batch_size: 32 | global_step: 217 | reduced_train_loss: 0.2656 | train_step_timing in s: 2.9 | consumed_samples: 6976 | val_loss: 0.279
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:51 pl:222] Successfully saved checkpoint from iteration     210 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.28-step=209-consumed_samples=6720.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:51 model_checkpoint:657] Async checkpoint save for step 210 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.28-step=209-consumed_samples=6720.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:55:51 dist_ckpt_io:160] Async finalization time took 0.059 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 218/999 | lr: 4.624e-06 | global_batch_size: 32 | global_step: 218 | reduced_train_loss: 0.2458 | train_step_timing in s: 2.892 | consumed_samples: 7008 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 219/999 | lr: 4.62e-06 | global_batch_size: 32 | global_step: 219 | reduced_train_loss: 0.2884 | train_step_timing in s: 2.895 | consumed_samples: 7040 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 220/999 | lr: 4.615e-06 | global_batch_size: 32 | global_step: 220 | reduced_train_loss: 0.2942 | train_step_timing in s: 2.899 | consumed_samples: 7072 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 221/999 | lr: 4.611e-06 | global_batch_size: 32 | global_step: 221 | reduced_train_loss: 0.2606 | train_step_timing in s: 2.903 | consumed_samples: 7104 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 222/999 | lr: 4.606e-06 | global_batch_size: 32 | global_step: 222 | reduced_train_loss: 0.3069 | train_step_timing in s: 2.898 | consumed_samples: 7136 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 223/999 | lr: 4.602e-06 | global_batch_size: 32 | global_step: 223 | reduced_train_loss: 0.3062 | train_step_timing in s: 2.898 | consumed_samples: 7168 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 224/999 | lr: 4.597e-06 | global_batch_size: 32 | global_step: 224 | reduced_train_loss: 0.3655 | train_step_timing in s: 2.9 | consumed_samples: 7200 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 225/999 | lr: 4.593e-06 | global_batch_size: 32 | global_step: 225 | reduced_train_loss: 0.3837 | train_step_timing in s: 2.899 | consumed_samples: 7232 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 226/999 | lr: 4.588e-06 | global_batch_size: 32 | global_step: 226 | reduced_train_loss: 0.248 | train_step_timing in s: 2.896 | consumed_samples: 7264 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 227/999 | lr: 4.584e-06 | global_batch_size: 32 | global_step: 227 | reduced_train_loss: 0.2713 | train_step_timing in s: 2.901 | consumed_samples: 7296 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 228/999 | lr: 4.579e-06 | global_batch_size: 32 | global_step: 228 | reduced_train_loss: 0.3593 | train_step_timing in s: 2.901 | consumed_samples: 7328 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 229/999 | lr: 4.575e-06 | global_batch_size: 32 | global_step: 229 | reduced_train_loss: 0.2952 | train_step_timing in s: 2.902 | consumed_samples: 7360 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 230/999 | lr: 4.57e-06 | global_batch_size: 32 | global_step: 230 | reduced_train_loss: 0.2829 | train_step_timing in s: 2.898 | consumed_samples: 7392 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 231/999 | lr: 4.565e-06 | global_batch_size: 32 | global_step: 231 | reduced_train_loss: 0.3418 | train_step_timing in s: 2.906 | consumed_samples: 7424 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 232/999 | lr: 4.561e-06 | global_batch_size: 32 | global_step: 232 | reduced_train_loss: 0.2399 | train_step_timing in s: 2.901 | consumed_samples: 7456 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 233/999 | lr: 4.556e-06 | global_batch_size: 32 | global_step: 233 | reduced_train_loss: 0.5461 | train_step_timing in s: 2.896 | consumed_samples: 7488 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 234/999 | lr: 4.551e-06 | global_batch_size: 32 | global_step: 234 | reduced_train_loss: 0.3471 | train_step_timing in s: 2.9 | consumed_samples: 7520 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 235/999 | lr: 4.547e-06 | global_batch_size: 32 | global_step: 235 | reduced_train_loss: 0.3029 | train_step_timing in s: 2.893 | consumed_samples: 7552 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 236/999 | lr: 4.542e-06 | global_batch_size: 32 | global_step: 236 | reduced_train_loss: 0.2833 | train_step_timing in s: 2.9 | consumed_samples: 7584 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 237/999 | lr: 4.537e-06 | global_batch_size: 32 | global_step: 237 | reduced_train_loss: 0.2352 | train_step_timing in s: 2.897 | consumed_samples: 7616 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 238/999 | lr: 4.532e-06 | global_batch_size: 32 | global_step: 238 | reduced_train_loss: 0.2472 | train_step_timing in s: 2.901 | consumed_samples: 7648 | val_loss: 0.279
s_act_ckpt/0 [default0]:Training epoch 0, iteration 239/999 | lr: 4.527e-06 | global_batch_size: 32 | global_step: 239 | reduced_train_loss: 0.2796 | train_step_timing in s: 2.9 | consumed_samples: 7680 | val_loss: 0.279
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:22 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 240 : Start time: 1764082641.821s : Save duration: 0.187s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:22 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.27-step=239-consumed_samples=7680.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 240/999 | lr: 4.523e-06 | global_batch_size: 32 | global_step: 240 | reduced_train_loss: 0.2464 | train_step_timing in s: 2.913 | consumed_samples: 7712 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:25 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 241/999 | lr: 4.518e-06 | global_batch_size: 32 | global_step: 241 | reduced_train_loss: 0.2857 | train_step_timing in s: 2.893 | consumed_samples: 7744 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:28 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 242/999 | lr: 4.513e-06 | global_batch_size: 32 | global_step: 242 | reduced_train_loss: 0.3232 | train_step_timing in s: 2.904 | consumed_samples: 7776 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:31 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 243/999 | lr: 4.508e-06 | global_batch_size: 32 | global_step: 243 | reduced_train_loss: 0.2789 | train_step_timing in s: 2.896 | consumed_samples: 7808 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:34 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 244/999 | lr: 4.503e-06 | global_batch_size: 32 | global_step: 244 | reduced_train_loss: 0.3223 | train_step_timing in s: 2.903 | consumed_samples: 7840 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:37 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 245/999 | lr: 4.498e-06 | global_batch_size: 32 | global_step: 245 | reduced_train_loss: 0.288 | train_step_timing in s: 2.891 | consumed_samples: 7872 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:40 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 246/999 | lr: 4.493e-06 | global_batch_size: 32 | global_step: 246 | reduced_train_loss: 0.2585 | train_step_timing in s: 2.889 | consumed_samples: 7904 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:43 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 247/999 | lr: 4.488e-06 | global_batch_size: 32 | global_step: 247 | reduced_train_loss: 0.2671 | train_step_timing in s: 2.893 | consumed_samples: 7936 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:45 pl:222] Successfully saved checkpoint from iteration     240 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.27-step=239-consumed_samples=7680.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:45 model_checkpoint:657] Async checkpoint save for step 240 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.27-step=239-consumed_samples=7680.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:46 dist_ckpt_io:160] Async finalization time took 0.748 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 248/999 | lr: 4.483e-06 | global_batch_size: 32 | global_step: 248 | reduced_train_loss: 0.3044 | train_step_timing in s: 2.891 | consumed_samples: 7968 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 249/999 | lr: 4.478e-06 | global_batch_size: 32 | global_step: 249 | reduced_train_loss: 0.2927 | train_step_timing in s: 2.889 | consumed_samples: 8000 | val_loss: 0.272
s_act_ckpt/0 [default0]:Epoch 0, global step 249: 'val_loss' reached 0.27201 (best 0.27201), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.27-step=249-consumed_samples=8000.0.ckpt' as top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:53 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 249 : Start time: 1764082672.553s : Save duration: 0.646s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:53 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.27-step=249-consumed_samples=8000.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:53 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 250/999 | lr: 4.473e-06 | global_batch_size: 32 | global_step: 250 | reduced_train_loss: 0.2684 | train_step_timing in s: 2.89 | consumed_samples: 8032 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:56 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 251/999 | lr: 4.468e-06 | global_batch_size: 32 | global_step: 251 | reduced_train_loss: 0.3171 | train_step_timing in s: 2.892 | consumed_samples: 8064 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:57:59 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 252/999 | lr: 4.463e-06 | global_batch_size: 32 | global_step: 252 | reduced_train_loss: 0.2363 | train_step_timing in s: 2.901 | consumed_samples: 8096 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:02 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 253/999 | lr: 4.458e-06 | global_batch_size: 32 | global_step: 253 | reduced_train_loss: 0.299 | train_step_timing in s: 2.896 | consumed_samples: 8128 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:05 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 254/999 | lr: 4.452e-06 | global_batch_size: 32 | global_step: 254 | reduced_train_loss: 0.2893 | train_step_timing in s: 2.906 | consumed_samples: 8160 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:08 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 255/999 | lr: 4.447e-06 | global_batch_size: 32 | global_step: 255 | reduced_train_loss: 0.3583 | train_step_timing in s: 2.899 | consumed_samples: 8192 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:11 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 256/999 | lr: 4.442e-06 | global_batch_size: 32 | global_step: 256 | reduced_train_loss: 0.2341 | train_step_timing in s: 2.899 | consumed_samples: 8224 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:14 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 257/999 | lr: 4.437e-06 | global_batch_size: 32 | global_step: 257 | reduced_train_loss: 0.236 | train_step_timing in s: 2.896 | consumed_samples: 8256 | val_loss: 0.272
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:17 pl:222] Successfully saved checkpoint from iteration     249 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.27-step=249-consumed_samples=8000.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:17 model_checkpoint:657] Async checkpoint save for step 250 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.27-step=249-consumed_samples=8000.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:58:18 dist_ckpt_io:160] Async finalization time took 1.224 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 258/999 | lr: 4.432e-06 | global_batch_size: 32 | global_step: 258 | reduced_train_loss: 0.2768 | train_step_timing in s: 2.897 | consumed_samples: 8288 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 259/999 | lr: 4.426e-06 | global_batch_size: 32 | global_step: 259 | reduced_train_loss: 0.2784 | train_step_timing in s: 2.899 | consumed_samples: 8320 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 260/999 | lr: 4.421e-06 | global_batch_size: 32 | global_step: 260 | reduced_train_loss: 0.1914 | train_step_timing in s: 2.896 | consumed_samples: 8352 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 261/999 | lr: 4.416e-06 | global_batch_size: 32 | global_step: 261 | reduced_train_loss: 0.3269 | train_step_timing in s: 2.907 | consumed_samples: 8384 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 262/999 | lr: 4.41e-06 | global_batch_size: 32 | global_step: 262 | reduced_train_loss: 0.2513 | train_step_timing in s: 2.899 | consumed_samples: 8416 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 263/999 | lr: 4.405e-06 | global_batch_size: 32 | global_step: 263 | reduced_train_loss: 0.2543 | train_step_timing in s: 2.898 | consumed_samples: 8448 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 264/999 | lr: 4.4e-06 | global_batch_size: 32 | global_step: 264 | reduced_train_loss: 0.2295 | train_step_timing in s: 2.893 | consumed_samples: 8480 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 265/999 | lr: 4.394e-06 | global_batch_size: 32 | global_step: 265 | reduced_train_loss: 0.2811 | train_step_timing in s: 2.899 | consumed_samples: 8512 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 266/999 | lr: 4.389e-06 | global_batch_size: 32 | global_step: 266 | reduced_train_loss: 0.2877 | train_step_timing in s: 2.898 | consumed_samples: 8544 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 267/999 | lr: 4.383e-06 | global_batch_size: 32 | global_step: 267 | reduced_train_loss: 0.3357 | train_step_timing in s: 2.895 | consumed_samples: 8576 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 268/999 | lr: 4.378e-06 | global_batch_size: 32 | global_step: 268 | reduced_train_loss: 0.2541 | train_step_timing in s: 2.897 | consumed_samples: 8608 | val_loss: 0.272
s_act_ckpt/0 [default0]:Training epoch 0, iteration 269/999 | lr: 4.373e-06 | global_batch_size: 32 | global_step: 269 | reduced_train_loss: 0.3983 | train_step_timing in s: 2.9 | consumed_samples: 8640 | val_loss: 0.272
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:20 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 270 : Start time: 1764082760.144s : Save duration: 0.190s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:20 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=269-consumed_samples=8640.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 270/999 | lr: 4.367e-06 | global_batch_size: 32 | global_step: 270 | reduced_train_loss: 0.3002 | train_step_timing in s: 2.907 | consumed_samples: 8672 | val_loss: 0.2645
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:23 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 271/999 | lr: 4.362e-06 | global_batch_size: 32 | global_step: 271 | reduced_train_loss: 0.2875 | train_step_timing in s: 2.886 | consumed_samples: 8704 | val_loss: 0.2645
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:26 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 272/999 | lr: 4.356e-06 | global_batch_size: 32 | global_step: 272 | reduced_train_loss: 0.2533 | train_step_timing in s: 2.895 | consumed_samples: 8736 | val_loss: 0.2645
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:29 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 273/999 | lr: 4.35e-06 | global_batch_size: 32 | global_step: 273 | reduced_train_loss: 0.2193 | train_step_timing in s: 2.896 | consumed_samples: 8768 | val_loss: 0.2645
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:32 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 274/999 | lr: 4.345e-06 | global_batch_size: 32 | global_step: 274 | reduced_train_loss: 0.2227 | train_step_timing in s: 2.897 | consumed_samples: 8800 | val_loss: 0.2645
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:35 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 275/999 | lr: 4.339e-06 | global_batch_size: 32 | global_step: 275 | reduced_train_loss: 0.2981 | train_step_timing in s: 2.893 | consumed_samples: 8832 | val_loss: 0.2645
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:38 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 276/999 | lr: 4.334e-06 | global_batch_size: 32 | global_step: 276 | reduced_train_loss: 0.3994 | train_step_timing in s: 2.907 | consumed_samples: 8864 | val_loss: 0.2645
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:41 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 277/999 | lr: 4.328e-06 | global_batch_size: 32 | global_step: 277 | reduced_train_loss: 0.2214 | train_step_timing in s: 2.895 | consumed_samples: 8896 | val_loss: 0.2645
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:44 pl:222] Successfully saved checkpoint from iteration     270 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=269-consumed_samples=8640.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:44 model_checkpoint:657] Async checkpoint save for step 270 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=269-consumed_samples=8640.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 15:59:44 dist_ckpt_io:160] Async finalization time took 0.062 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 278/999 | lr: 4.322e-06 | global_batch_size: 32 | global_step: 278 | reduced_train_loss: 0.216 | train_step_timing in s: 2.895 | consumed_samples: 8928 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 279/999 | lr: 4.317e-06 | global_batch_size: 32 | global_step: 279 | reduced_train_loss: 0.3379 | train_step_timing in s: 2.899 | consumed_samples: 8960 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 280/999 | lr: 4.311e-06 | global_batch_size: 32 | global_step: 280 | reduced_train_loss: 0.2724 | train_step_timing in s: 2.909 | consumed_samples: 8992 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 281/999 | lr: 4.305e-06 | global_batch_size: 32 | global_step: 281 | reduced_train_loss: 0.2596 | train_step_timing in s: 2.898 | consumed_samples: 9024 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 282/999 | lr: 4.3e-06 | global_batch_size: 32 | global_step: 282 | reduced_train_loss: 0.2565 | train_step_timing in s: 2.893 | consumed_samples: 9056 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 283/999 | lr: 4.294e-06 | global_batch_size: 32 | global_step: 283 | reduced_train_loss: 0.2782 | train_step_timing in s: 2.894 | consumed_samples: 9088 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 284/999 | lr: 4.288e-06 | global_batch_size: 32 | global_step: 284 | reduced_train_loss: 0.2467 | train_step_timing in s: 2.901 | consumed_samples: 9120 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 285/999 | lr: 4.282e-06 | global_batch_size: 32 | global_step: 285 | reduced_train_loss: 0.274 | train_step_timing in s: 2.894 | consumed_samples: 9152 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 286/999 | lr: 4.277e-06 | global_batch_size: 32 | global_step: 286 | reduced_train_loss: 0.2565 | train_step_timing in s: 2.899 | consumed_samples: 9184 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 287/999 | lr: 4.271e-06 | global_batch_size: 32 | global_step: 287 | reduced_train_loss: 0.2976 | train_step_timing in s: 2.903 | consumed_samples: 9216 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 288/999 | lr: 4.265e-06 | global_batch_size: 32 | global_step: 288 | reduced_train_loss: 0.3649 | train_step_timing in s: 2.9 | consumed_samples: 9248 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 289/999 | lr: 4.259e-06 | global_batch_size: 32 | global_step: 289 | reduced_train_loss: 0.2976 | train_step_timing in s: 2.898 | consumed_samples: 9280 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 290/999 | lr: 4.253e-06 | global_batch_size: 32 | global_step: 290 | reduced_train_loss: 0.3027 | train_step_timing in s: 2.9 | consumed_samples: 9312 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 291/999 | lr: 4.247e-06 | global_batch_size: 32 | global_step: 291 | reduced_train_loss: 0.2144 | train_step_timing in s: 2.899 | consumed_samples: 9344 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 292/999 | lr: 4.241e-06 | global_batch_size: 32 | global_step: 292 | reduced_train_loss: 0.2331 | train_step_timing in s: 2.899 | consumed_samples: 9376 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 293/999 | lr: 4.235e-06 | global_batch_size: 32 | global_step: 293 | reduced_train_loss: 0.2396 | train_step_timing in s: 2.898 | consumed_samples: 9408 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 294/999 | lr: 4.229e-06 | global_batch_size: 32 | global_step: 294 | reduced_train_loss: 0.2691 | train_step_timing in s: 2.902 | consumed_samples: 9440 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 295/999 | lr: 4.223e-06 | global_batch_size: 32 | global_step: 295 | reduced_train_loss: 0.2468 | train_step_timing in s: 2.903 | consumed_samples: 9472 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 296/999 | lr: 4.217e-06 | global_batch_size: 32 | global_step: 296 | reduced_train_loss: 0.2794 | train_step_timing in s: 2.903 | consumed_samples: 9504 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 297/999 | lr: 4.211e-06 | global_batch_size: 32 | global_step: 297 | reduced_train_loss: 0.2557 | train_step_timing in s: 2.896 | consumed_samples: 9536 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 298/999 | lr: 4.205e-06 | global_batch_size: 32 | global_step: 298 | reduced_train_loss: 0.2728 | train_step_timing in s: 2.898 | consumed_samples: 9568 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Training epoch 0, iteration 299/999 | lr: 4.199e-06 | global_batch_size: 32 | global_step: 299 | reduced_train_loss: 0.2937 | train_step_timing in s: 2.9 | consumed_samples: 9600 | val_loss: 0.2645
s_act_ckpt/0 [default0]:Epoch 0, global step 299: 'val_loss' reached 0.26455 (best 0.26455), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=299-consumed_samples=9600.0.ckpt' as top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:00:48 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 299 : Start time: 1764082848.510s : Save duration: 0.177s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:00:49 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=299-consumed_samples=9600.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:00:49 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:Training epoch 0, iteration 300/999 | lr: 4.193e-06 | global_batch_size: 32 | global_step: 300 | reduced_train_loss: 0.3008 | train_step_timing in s: 2.893 | consumed_samples: 9632 | val_loss: 0.2591
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:01:18 pl:222] Successfully saved checkpoint from iteration     299 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=299-consumed_samples=9600.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:01:18 model_checkpoint:657] Async checkpoint save for step 300 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=299-consumed_samples=9600.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:01:20 dist_ckpt_io:160] Async finalization time took 1.213 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 301/999 | lr: 4.187e-06 | global_batch_size: 32 | global_step: 301 | reduced_train_loss: 0.5415 | train_step_timing in s: 2.894 | consumed_samples: 9664 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 302/999 | lr: 4.181e-06 | global_batch_size: 32 | global_step: 302 | reduced_train_loss: 0.2829 | train_step_timing in s: 2.895 | consumed_samples: 9696 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 303/999 | lr: 4.175e-06 | global_batch_size: 32 | global_step: 303 | reduced_train_loss: 0.3423 | train_step_timing in s: 2.89 | consumed_samples: 9728 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 304/999 | lr: 4.169e-06 | global_batch_size: 32 | global_step: 304 | reduced_train_loss: 0.2725 | train_step_timing in s: 2.898 | consumed_samples: 9760 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 305/999 | lr: 4.163e-06 | global_batch_size: 32 | global_step: 305 | reduced_train_loss: 0.2176 | train_step_timing in s: 2.895 | consumed_samples: 9792 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 306/999 | lr: 4.156e-06 | global_batch_size: 32 | global_step: 306 | reduced_train_loss: 0.2067 | train_step_timing in s: 2.893 | consumed_samples: 9824 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 307/999 | lr: 4.15e-06 | global_batch_size: 32 | global_step: 307 | reduced_train_loss: 0.2446 | train_step_timing in s: 2.895 | consumed_samples: 9856 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 308/999 | lr: 4.144e-06 | global_batch_size: 32 | global_step: 308 | reduced_train_loss: 0.2943 | train_step_timing in s: 2.899 | consumed_samples: 9888 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 309/999 | lr: 4.138e-06 | global_batch_size: 32 | global_step: 309 | reduced_train_loss: 0.2715 | train_step_timing in s: 2.894 | consumed_samples: 9920 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 310/999 | lr: 4.131e-06 | global_batch_size: 32 | global_step: 310 | reduced_train_loss: 0.1866 | train_step_timing in s: 2.904 | consumed_samples: 9952 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 311/999 | lr: 4.125e-06 | global_batch_size: 32 | global_step: 311 | reduced_train_loss: 0.2547 | train_step_timing in s: 2.899 | consumed_samples: 9984 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 312/999 | lr: 4.119e-06 | global_batch_size: 32 | global_step: 312 | reduced_train_loss: 0.2805 | train_step_timing in s: 2.891 | consumed_samples: 10016 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 313/999 | lr: 4.113e-06 | global_batch_size: 32 | global_step: 313 | reduced_train_loss: 0.3346 | train_step_timing in s: 2.898 | consumed_samples: 10048 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 314/999 | lr: 4.106e-06 | global_batch_size: 32 | global_step: 314 | reduced_train_loss: 0.4411 | train_step_timing in s: 2.901 | consumed_samples: 10080 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 315/999 | lr: 4.1e-06 | global_batch_size: 32 | global_step: 315 | reduced_train_loss: 0.2681 | train_step_timing in s: 2.903 | consumed_samples: 10112 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 316/999 | lr: 4.094e-06 | global_batch_size: 32 | global_step: 316 | reduced_train_loss: 0.2509 | train_step_timing in s: 2.898 | consumed_samples: 10144 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 317/999 | lr: 4.087e-06 | global_batch_size: 32 | global_step: 317 | reduced_train_loss: 0.2321 | train_step_timing in s: 2.898 | consumed_samples: 10176 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 318/999 | lr: 4.081e-06 | global_batch_size: 32 | global_step: 318 | reduced_train_loss: 0.2602 | train_step_timing in s: 2.902 | consumed_samples: 10208 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 319/999 | lr: 4.074e-06 | global_batch_size: 32 | global_step: 319 | reduced_train_loss: 0.3221 | train_step_timing in s: 2.903 | consumed_samples: 10240 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 320/999 | lr: 4.068e-06 | global_batch_size: 32 | global_step: 320 | reduced_train_loss: 0.246 | train_step_timing in s: 2.897 | consumed_samples: 10272 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 321/999 | lr: 4.061e-06 | global_batch_size: 32 | global_step: 321 | reduced_train_loss: 0.3402 | train_step_timing in s: 2.901 | consumed_samples: 10304 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 322/999 | lr: 4.055e-06 | global_batch_size: 32 | global_step: 322 | reduced_train_loss: 0.3204 | train_step_timing in s: 2.897 | consumed_samples: 10336 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 323/999 | lr: 4.049e-06 | global_batch_size: 32 | global_step: 323 | reduced_train_loss: 0.3162 | train_step_timing in s: 2.902 | consumed_samples: 10368 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 324/999 | lr: 4.042e-06 | global_batch_size: 32 | global_step: 324 | reduced_train_loss: 0.2979 | train_step_timing in s: 2.904 | consumed_samples: 10400 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 325/999 | lr: 4.036e-06 | global_batch_size: 32 | global_step: 325 | reduced_train_loss: 0.2164 | train_step_timing in s: 2.902 | consumed_samples: 10432 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 326/999 | lr: 4.029e-06 | global_batch_size: 32 | global_step: 326 | reduced_train_loss: 0.2931 | train_step_timing in s: 2.901 | consumed_samples: 10464 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 327/999 | lr: 4.022e-06 | global_batch_size: 32 | global_step: 327 | reduced_train_loss: 0.2586 | train_step_timing in s: 2.904 | consumed_samples: 10496 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 328/999 | lr: 4.016e-06 | global_batch_size: 32 | global_step: 328 | reduced_train_loss: 0.2055 | train_step_timing in s: 2.901 | consumed_samples: 10528 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Training epoch 0, iteration 329/999 | lr: 4.009e-06 | global_batch_size: 32 | global_step: 329 | reduced_train_loss: 0.2689 | train_step_timing in s: 2.901 | consumed_samples: 10560 | val_loss: 0.2591
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:12 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 330 : Start time: 1764082992.206s : Save duration: 0.180s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:12 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=329-consumed_samples=10560.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 330/999 | lr: 4.003e-06 | global_batch_size: 32 | global_step: 330 | reduced_train_loss: 0.3491 | train_step_timing in s: 2.904 | consumed_samples: 10592 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:15 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 331/999 | lr: 3.996e-06 | global_batch_size: 32 | global_step: 331 | reduced_train_loss: 0.2789 | train_step_timing in s: 2.893 | consumed_samples: 10624 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:18 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 332/999 | lr: 3.989e-06 | global_batch_size: 32 | global_step: 332 | reduced_train_loss: 0.1986 | train_step_timing in s: 2.893 | consumed_samples: 10656 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:21 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 333/999 | lr: 3.983e-06 | global_batch_size: 32 | global_step: 333 | reduced_train_loss: 0.2859 | train_step_timing in s: 2.898 | consumed_samples: 10688 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:24 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 334/999 | lr: 3.976e-06 | global_batch_size: 32 | global_step: 334 | reduced_train_loss: 0.2094 | train_step_timing in s: 2.896 | consumed_samples: 10720 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:27 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 335/999 | lr: 3.969e-06 | global_batch_size: 32 | global_step: 335 | reduced_train_loss: 0.3096 | train_step_timing in s: 2.9 | consumed_samples: 10752 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:30 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 336/999 | lr: 3.963e-06 | global_batch_size: 32 | global_step: 336 | reduced_train_loss: 0.2947 | train_step_timing in s: 2.9 | consumed_samples: 10784 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:33 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 337/999 | lr: 3.956e-06 | global_batch_size: 32 | global_step: 337 | reduced_train_loss: 0.2601 | train_step_timing in s: 2.901 | consumed_samples: 10816 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:36 pl:222] Successfully saved checkpoint from iteration     330 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=329-consumed_samples=10560.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:36 model_checkpoint:657] Async checkpoint save for step 330 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=329-consumed_samples=10560.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:03:36 dist_ckpt_io:160] Async finalization time took 0.056 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 338/999 | lr: 3.949e-06 | global_batch_size: 32 | global_step: 338 | reduced_train_loss: 0.307 | train_step_timing in s: 2.9 | consumed_samples: 10848 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 339/999 | lr: 3.943e-06 | global_batch_size: 32 | global_step: 339 | reduced_train_loss: 0.2296 | train_step_timing in s: 2.901 | consumed_samples: 10880 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 340/999 | lr: 3.936e-06 | global_batch_size: 32 | global_step: 340 | reduced_train_loss: 0.2384 | train_step_timing in s: 2.896 | consumed_samples: 10912 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 341/999 | lr: 3.929e-06 | global_batch_size: 32 | global_step: 341 | reduced_train_loss: 0.2612 | train_step_timing in s: 2.906 | consumed_samples: 10944 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 342/999 | lr: 3.922e-06 | global_batch_size: 32 | global_step: 342 | reduced_train_loss: 0.3006 | train_step_timing in s: 2.893 | consumed_samples: 10976 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 343/999 | lr: 3.915e-06 | global_batch_size: 32 | global_step: 343 | reduced_train_loss: 0.2686 | train_step_timing in s: 2.9 | consumed_samples: 11008 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 344/999 | lr: 3.909e-06 | global_batch_size: 32 | global_step: 344 | reduced_train_loss: 0.1855 | train_step_timing in s: 2.9 | consumed_samples: 11040 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 345/999 | lr: 3.902e-06 | global_batch_size: 32 | global_step: 345 | reduced_train_loss: 0.2406 | train_step_timing in s: 2.902 | consumed_samples: 11072 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 346/999 | lr: 3.895e-06 | global_batch_size: 32 | global_step: 346 | reduced_train_loss: 0.2974 | train_step_timing in s: 2.907 | consumed_samples: 11104 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 347/999 | lr: 3.888e-06 | global_batch_size: 32 | global_step: 347 | reduced_train_loss: 0.3472 | train_step_timing in s: 2.9 | consumed_samples: 11136 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 348/999 | lr: 3.881e-06 | global_batch_size: 32 | global_step: 348 | reduced_train_loss: 0.3219 | train_step_timing in s: 2.904 | consumed_samples: 11168 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 349/999 | lr: 3.874e-06 | global_batch_size: 32 | global_step: 349 | reduced_train_loss: 0.2925 | train_step_timing in s: 2.896 | consumed_samples: 11200 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Epoch 0, global step 349: 'val_loss' reached 0.25545 (best 0.25545), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=349-consumed_samples=11200.0.ckpt' as top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:12 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 349 : Start time: 1764083051.945s : Save duration: 0.177s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:12 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=349-consumed_samples=11200.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:12 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 350/999 | lr: 3.867e-06 | global_batch_size: 32 | global_step: 350 | reduced_train_loss: 0.2215 | train_step_timing in s: 2.892 | consumed_samples: 11232 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:15 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 351/999 | lr: 3.86e-06 | global_batch_size: 32 | global_step: 351 | reduced_train_loss: 0.2407 | train_step_timing in s: 2.896 | consumed_samples: 11264 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:18 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 352/999 | lr: 3.853e-06 | global_batch_size: 32 | global_step: 352 | reduced_train_loss: 0.2456 | train_step_timing in s: 2.897 | consumed_samples: 11296 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:21 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 353/999 | lr: 3.847e-06 | global_batch_size: 32 | global_step: 353 | reduced_train_loss: 0.3467 | train_step_timing in s: 2.895 | consumed_samples: 11328 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:24 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 354/999 | lr: 3.84e-06 | global_batch_size: 32 | global_step: 354 | reduced_train_loss: 0.2584 | train_step_timing in s: 2.896 | consumed_samples: 11360 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:27 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 355/999 | lr: 3.833e-06 | global_batch_size: 32 | global_step: 355 | reduced_train_loss: 0.32 | train_step_timing in s: 2.902 | consumed_samples: 11392 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:30 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 356/999 | lr: 3.826e-06 | global_batch_size: 32 | global_step: 356 | reduced_train_loss: 0.3161 | train_step_timing in s: 2.901 | consumed_samples: 11424 | val_loss: 0.2555
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:33 pl:222] Successfully saved checkpoint from iteration     349 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=349-consumed_samples=11200.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:33 model_checkpoint:657] Async checkpoint save for step 350 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.26-step=349-consumed_samples=11200.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:04:34 dist_ckpt_io:160] Async finalization time took 1.389 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 357/999 | lr: 3.819e-06 | global_batch_size: 32 | global_step: 357 | reduced_train_loss: 0.2689 | train_step_timing in s: 2.896 | consumed_samples: 11456 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 358/999 | lr: 3.812e-06 | global_batch_size: 32 | global_step: 358 | reduced_train_loss: 0.1785 | train_step_timing in s: 2.899 | consumed_samples: 11488 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Training epoch 0, iteration 359/999 | lr: 3.804e-06 | global_batch_size: 32 | global_step: 359 | reduced_train_loss: 0.2544 | train_step_timing in s: 2.896 | consumed_samples: 11520 | val_loss: 0.2555
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:10 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 360 : Start time: 1764083110.690s : Save duration: 0.186s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:11 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=359-consumed_samples=11520.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 360/999 | lr: 3.797e-06 | global_batch_size: 32 | global_step: 360 | reduced_train_loss: 0.3376 | train_step_timing in s: 2.912 | consumed_samples: 11552 | val_loss: 0.2533
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:14 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 361/999 | lr: 3.79e-06 | global_batch_size: 32 | global_step: 361 | reduced_train_loss: 0.2309 | train_step_timing in s: 2.893 | consumed_samples: 11584 | val_loss: 0.2533
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:17 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 362/999 | lr: 3.783e-06 | global_batch_size: 32 | global_step: 362 | reduced_train_loss: 0.2328 | train_step_timing in s: 2.898 | consumed_samples: 11616 | val_loss: 0.2533
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:20 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 363/999 | lr: 3.776e-06 | global_batch_size: 32 | global_step: 363 | reduced_train_loss: 0.3447 | train_step_timing in s: 2.895 | consumed_samples: 11648 | val_loss: 0.2533
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:23 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 364/999 | lr: 3.769e-06 | global_batch_size: 32 | global_step: 364 | reduced_train_loss: 0.2666 | train_step_timing in s: 2.898 | consumed_samples: 11680 | val_loss: 0.2533
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:26 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 365/999 | lr: 3.762e-06 | global_batch_size: 32 | global_step: 365 | reduced_train_loss: 0.3478 | train_step_timing in s: 2.895 | consumed_samples: 11712 | val_loss: 0.2533
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:28 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 366/999 | lr: 3.755e-06 | global_batch_size: 32 | global_step: 366 | reduced_train_loss: 0.2078 | train_step_timing in s: 2.897 | consumed_samples: 11744 | val_loss: 0.2533
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:31 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 367/999 | lr: 3.748e-06 | global_batch_size: 32 | global_step: 367 | reduced_train_loss: 0.1857 | train_step_timing in s: 2.897 | consumed_samples: 11776 | val_loss: 0.2533
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:34 pl:222] Successfully saved checkpoint from iteration     360 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=359-consumed_samples=11520.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:34 model_checkpoint:657] Async checkpoint save for step 360 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=359-consumed_samples=11520.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:05:34 dist_ckpt_io:160] Async finalization time took 0.056 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 368/999 | lr: 3.74e-06 | global_batch_size: 32 | global_step: 368 | reduced_train_loss: 0.1932 | train_step_timing in s: 2.893 | consumed_samples: 11808 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 369/999 | lr: 3.733e-06 | global_batch_size: 32 | global_step: 369 | reduced_train_loss: 0.2983 | train_step_timing in s: 2.893 | consumed_samples: 11840 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 370/999 | lr: 3.726e-06 | global_batch_size: 32 | global_step: 370 | reduced_train_loss: 0.303 | train_step_timing in s: 2.894 | consumed_samples: 11872 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 371/999 | lr: 3.719e-06 | global_batch_size: 32 | global_step: 371 | reduced_train_loss: 0.3776 | train_step_timing in s: 2.899 | consumed_samples: 11904 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 372/999 | lr: 3.712e-06 | global_batch_size: 32 | global_step: 372 | reduced_train_loss: 0.2483 | train_step_timing in s: 2.895 | consumed_samples: 11936 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 373/999 | lr: 3.704e-06 | global_batch_size: 32 | global_step: 373 | reduced_train_loss: 0.2243 | train_step_timing in s: 2.901 | consumed_samples: 11968 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 374/999 | lr: 3.697e-06 | global_batch_size: 32 | global_step: 374 | reduced_train_loss: 0.1989 | train_step_timing in s: 2.899 | consumed_samples: 12000 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 375/999 | lr: 3.69e-06 | global_batch_size: 32 | global_step: 375 | reduced_train_loss: 0.3975 | train_step_timing in s: 2.898 | consumed_samples: 12032 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 376/999 | lr: 3.683e-06 | global_batch_size: 32 | global_step: 376 | reduced_train_loss: 0.1787 | train_step_timing in s: 2.895 | consumed_samples: 12064 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 377/999 | lr: 3.675e-06 | global_batch_size: 32 | global_step: 377 | reduced_train_loss: 0.2704 | train_step_timing in s: 2.899 | consumed_samples: 12096 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 378/999 | lr: 3.668e-06 | global_batch_size: 32 | global_step: 378 | reduced_train_loss: 0.2992 | train_step_timing in s: 2.898 | consumed_samples: 12128 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 379/999 | lr: 3.661e-06 | global_batch_size: 32 | global_step: 379 | reduced_train_loss: 0.2431 | train_step_timing in s: 2.902 | consumed_samples: 12160 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 380/999 | lr: 3.653e-06 | global_batch_size: 32 | global_step: 380 | reduced_train_loss: 0.2146 | train_step_timing in s: 2.893 | consumed_samples: 12192 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 381/999 | lr: 3.646e-06 | global_batch_size: 32 | global_step: 381 | reduced_train_loss: 0.2797 | train_step_timing in s: 2.899 | consumed_samples: 12224 | val_loss: 0.2533[default0]:
s_act_ckpt/0 [default0]:Training epoch 0, iteration 382/999 | lr: 3.639e-06 | global_batch_size: 32 | global_step: 382 | reduced_train_loss: 0.2462 | train_step_timing in s: 2.897 | consumed_samples: 12256 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 383/999 | lr: 3.631e-06 | global_batch_size: 32 | global_step: 383 | reduced_train_loss: 0.1852 | train_step_timing in s: 2.898 | consumed_samples: 12288 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 384/999 | lr: 3.624e-06 | global_batch_size: 32 | global_step: 384 | reduced_train_loss: 0.2061 | train_step_timing in s: 2.892 | consumed_samples: 12320 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 385/999 | lr: 3.617e-06 | global_batch_size: 32 | global_step: 385 | reduced_train_loss: 0.249 | train_step_timing in s: 2.899 | consumed_samples: 12352 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 386/999 | lr: 3.609e-06 | global_batch_size: 32 | global_step: 386 | reduced_train_loss: 0.2223 | train_step_timing in s: 2.895 | consumed_samples: 12384 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 387/999 | lr: 3.602e-06 | global_batch_size: 32 | global_step: 387 | reduced_train_loss: 0.3532 | train_step_timing in s: 2.896 | consumed_samples: 12416 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 388/999 | lr: 3.594e-06 | global_batch_size: 32 | global_step: 388 | reduced_train_loss: 0.2724 | train_step_timing in s: 2.891 | consumed_samples: 12448 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Training epoch 0, iteration 389/999 | lr: 3.587e-06 | global_batch_size: 32 | global_step: 389 | reduced_train_loss: 0.3403 | train_step_timing in s: 2.901 | consumed_samples: 12480 | val_loss: 0.2533
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:05 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 390 : Start time: 1764083225.406s : Save duration: 0.187s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:06 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=389-consumed_samples=12480.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 390/999 | lr: 3.579e-06 | global_batch_size: 32 | global_step: 390 | reduced_train_loss: 0.1923 | train_step_timing in s: 2.915 | consumed_samples: 12512 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:09 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 391/999 | lr: 3.572e-06 | global_batch_size: 32 | global_step: 391 | reduced_train_loss: 0.2165 | train_step_timing in s: 2.888 | consumed_samples: 12544 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:12 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 392/999 | lr: 3.564e-06 | global_batch_size: 32 | global_step: 392 | reduced_train_loss: 0.2572 | train_step_timing in s: 2.896 | consumed_samples: 12576 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:14 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 393/999 | lr: 3.557e-06 | global_batch_size: 32 | global_step: 393 | reduced_train_loss: 0.2334 | train_step_timing in s: 2.895 | consumed_samples: 12608 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:17 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 394/999 | lr: 3.549e-06 | global_batch_size: 32 | global_step: 394 | reduced_train_loss: 0.1875 | train_step_timing in s: 2.896 | consumed_samples: 12640 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:20 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 395/999 | lr: 3.542e-06 | global_batch_size: 32 | global_step: 395 | reduced_train_loss: 0.2454 | train_step_timing in s: 2.897 | consumed_samples: 12672 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:23 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 396/999 | lr: 3.534e-06 | global_batch_size: 32 | global_step: 396 | reduced_train_loss: 0.3184 | train_step_timing in s: 2.892 | consumed_samples: 12704 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:26 pl:222] Successfully saved checkpoint from iteration     390 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=389-consumed_samples=12480.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:26 model_checkpoint:657] Async checkpoint save for step 390 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=389-consumed_samples=12480.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:27 dist_ckpt_io:160] Async finalization time took 0.625 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 397/999 | lr: 3.527e-06 | global_batch_size: 32 | global_step: 397 | reduced_train_loss: 0.226 | train_step_timing in s: 2.896 | consumed_samples: 12736 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 398/999 | lr: 3.519e-06 | global_batch_size: 32 | global_step: 398 | reduced_train_loss: 0.2601 | train_step_timing in s: 2.901 | consumed_samples: 12768 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 399/999 | lr: 3.512e-06 | global_batch_size: 32 | global_step: 399 | reduced_train_loss: 0.2494 | train_step_timing in s: 2.9 | consumed_samples: 12800 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Epoch 0, global step 399: 'val_loss' reached 0.25137 (best 0.25137), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=399-consumed_samples=12800.0.ckpt' as top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:36 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 399 : Start time: 1764083256.030s : Save duration: 0.653s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:37 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=399-consumed_samples=12800.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:37 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 400/999 | lr: 3.504e-06 | global_batch_size: 32 | global_step: 400 | reduced_train_loss: 0.2381 | train_step_timing in s: 2.891 | consumed_samples: 12832 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:40 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 401/999 | lr: 3.497e-06 | global_batch_size: 32 | global_step: 401 | reduced_train_loss: 0.2015 | train_step_timing in s: 2.901 | consumed_samples: 12864 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:43 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 402/999 | lr: 3.489e-06 | global_batch_size: 32 | global_step: 402 | reduced_train_loss: 0.3451 | train_step_timing in s: 2.896 | consumed_samples: 12896 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:46 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 403/999 | lr: 3.481e-06 | global_batch_size: 32 | global_step: 403 | reduced_train_loss: 0.2166 | train_step_timing in s: 2.896 | consumed_samples: 12928 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:48 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 404/999 | lr: 3.474e-06 | global_batch_size: 32 | global_step: 404 | reduced_train_loss: 0.176 | train_step_timing in s: 2.9 | consumed_samples: 12960 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:51 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 405/999 | lr: 3.466e-06 | global_batch_size: 32 | global_step: 405 | reduced_train_loss: 0.2687 | train_step_timing in s: 2.9 | consumed_samples: 12992 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:54 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 406/999 | lr: 3.459e-06 | global_batch_size: 32 | global_step: 406 | reduced_train_loss: 0.2414 | train_step_timing in s: 2.894 | consumed_samples: 13024 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:07:57 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 407/999 | lr: 3.451e-06 | global_batch_size: 32 | global_step: 407 | reduced_train_loss: 0.2585 | train_step_timing in s: 2.905 | consumed_samples: 13056 | val_loss: 0.2514
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:08:00 pl:222] Successfully saved checkpoint from iteration     399 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=399-consumed_samples=12800.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:08:00 model_checkpoint:657] Async checkpoint save for step 400 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=399-consumed_samples=12800.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:08:01 dist_ckpt_io:160] Async finalization time took 1.312 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 408/999 | lr: 3.443e-06 | global_batch_size: 32 | global_step: 408 | reduced_train_loss: 0.2245 | train_step_timing in s: 2.889 | consumed_samples: 13088 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 409/999 | lr: 3.436e-06 | global_batch_size: 32 | global_step: 409 | reduced_train_loss: 0.218 | train_step_timing in s: 2.901 | consumed_samples: 13120 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 410/999 | lr: 3.428e-06 | global_batch_size: 32 | global_step: 410 | reduced_train_loss: 0.223 | train_step_timing in s: 2.899 | consumed_samples: 13152 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 411/999 | lr: 3.42e-06 | global_batch_size: 32 | global_step: 411 | reduced_train_loss: 0.3314 | train_step_timing in s: 2.89 | consumed_samples: 13184 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 412/999 | lr: 3.413e-06 | global_batch_size: 32 | global_step: 412 | reduced_train_loss: 0.192 | train_step_timing in s: 2.902 | consumed_samples: 13216 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 413/999 | lr: 3.405e-06 | global_batch_size: 32 | global_step: 413 | reduced_train_loss: 0.214 | train_step_timing in s: 2.9 | consumed_samples: 13248 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 414/999 | lr: 3.397e-06 | global_batch_size: 32 | global_step: 414 | reduced_train_loss: 0.3718 | train_step_timing in s: 2.897 | consumed_samples: 13280 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 415/999 | lr: 3.389e-06 | global_batch_size: 32 | global_step: 415 | reduced_train_loss: 0.2462 | train_step_timing in s: 2.901 | consumed_samples: 13312 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 416/999 | lr: 3.382e-06 | global_batch_size: 32 | global_step: 416 | reduced_train_loss: 0.2495 | train_step_timing in s: 2.898 | consumed_samples: 13344 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 417/999 | lr: 3.374e-06 | global_batch_size: 32 | global_step: 417 | reduced_train_loss: 0.171 | train_step_timing in s: 2.903 | consumed_samples: 13376 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 418/999 | lr: 3.366e-06 | global_batch_size: 32 | global_step: 418 | reduced_train_loss: 0.2466 | train_step_timing in s: 2.892 | consumed_samples: 13408 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Training epoch 0, iteration 419/999 | lr: 3.359e-06 | global_batch_size: 32 | global_step: 419 | reduced_train_loss: 0.233 | train_step_timing in s: 2.903 | consumed_samples: 13440 | val_loss: 0.2514
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:04 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 420 : Start time: 1764083344.412s : Save duration: 0.185s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:05 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=419-consumed_samples=13440.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 420/999 | lr: 3.351e-06 | global_batch_size: 32 | global_step: 420 | reduced_train_loss: 0.2944 | train_step_timing in s: 2.906 | consumed_samples: 13472 | val_loss: 0.2479
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:08 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 421/999 | lr: 3.343e-06 | global_batch_size: 32 | global_step: 421 | reduced_train_loss: 0.2638 | train_step_timing in s: 2.892 | consumed_samples: 13504 | val_loss: 0.2479
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:10 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 422/999 | lr: 3.335e-06 | global_batch_size: 32 | global_step: 422 | reduced_train_loss: 0.2001 | train_step_timing in s: 2.893 | consumed_samples: 13536 | val_loss: 0.2479
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:13 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 423/999 | lr: 3.327e-06 | global_batch_size: 32 | global_step: 423 | reduced_train_loss: 0.2753 | train_step_timing in s: 2.901 | consumed_samples: 13568 | val_loss: 0.2479
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:16 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 424/999 | lr: 3.32e-06 | global_batch_size: 32 | global_step: 424 | reduced_train_loss: 0.2133 | train_step_timing in s: 2.905 | consumed_samples: 13600 | val_loss: 0.2479
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:19 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 425/999 | lr: 3.312e-06 | global_batch_size: 32 | global_step: 425 | reduced_train_loss: 0.3347 | train_step_timing in s: 2.898 | consumed_samples: 13632 | val_loss: 0.2479
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:22 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 426/999 | lr: 3.304e-06 | global_batch_size: 32 | global_step: 426 | reduced_train_loss: 0.183 | train_step_timing in s: 2.897 | consumed_samples: 13664 | val_loss: 0.2479
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:25 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 427/999 | lr: 3.296e-06 | global_batch_size: 32 | global_step: 427 | reduced_train_loss: 0.2901 | train_step_timing in s: 2.901 | consumed_samples: 13696 | val_loss: 0.2479
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:28 pl:222] Successfully saved checkpoint from iteration     420 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=419-consumed_samples=13440.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:28 model_checkpoint:657] Async checkpoint save for step 420 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=419-consumed_samples=13440.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:09:28 dist_ckpt_io:160] Async finalization time took 0.058 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 428/999 | lr: 3.288e-06 | global_batch_size: 32 | global_step: 428 | reduced_train_loss: 0.2116 | train_step_timing in s: 2.894 | consumed_samples: 13728 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 429/999 | lr: 3.28e-06 | global_batch_size: 32 | global_step: 429 | reduced_train_loss: 0.2407 | train_step_timing in s: 2.902 | consumed_samples: 13760 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 430/999 | lr: 3.273e-06 | global_batch_size: 32 | global_step: 430 | reduced_train_loss: 0.1975 | train_step_timing in s: 2.903 | consumed_samples: 13792 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 431/999 | lr: 3.265e-06 | global_batch_size: 32 | global_step: 431 | reduced_train_loss: 0.2762 | train_step_timing in s: 2.904 | consumed_samples: 13824 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 432/999 | lr: 3.257e-06 | global_batch_size: 32 | global_step: 432 | reduced_train_loss: 0.1959 | train_step_timing in s: 2.899 | consumed_samples: 13856 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 433/999 | lr: 3.249e-06 | global_batch_size: 32 | global_step: 433 | reduced_train_loss: 0.1957 | train_step_timing in s: 2.899 | consumed_samples: 13888 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 434/999 | lr: 3.241e-06 | global_batch_size: 32 | global_step: 434 | reduced_train_loss: 0.2706 | train_step_timing in s: 2.9 | consumed_samples: 13920 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 435/999 | lr: 3.233e-06 | global_batch_size: 32 | global_step: 435 | reduced_train_loss: 0.4977 | train_step_timing in s: 2.901 | consumed_samples: 13952 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 436/999 | lr: 3.225e-06 | global_batch_size: 32 | global_step: 436 | reduced_train_loss: 0.3276 | train_step_timing in s: 2.903 | consumed_samples: 13984 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 437/999 | lr: 3.217e-06 | global_batch_size: 32 | global_step: 437 | reduced_train_loss: 0.2121 | train_step_timing in s: 2.903 | consumed_samples: 14016 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 438/999 | lr: 3.209e-06 | global_batch_size: 32 | global_step: 438 | reduced_train_loss: 0.2785 | train_step_timing in s: 2.895 | consumed_samples: 14048 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 439/999 | lr: 3.201e-06 | global_batch_size: 32 | global_step: 439 | reduced_train_loss: 0.2518 | train_step_timing in s: 2.9 | consumed_samples: 14080 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 440/999 | lr: 3.194e-06 | global_batch_size: 32 | global_step: 440 | reduced_train_loss: 0.2765 | train_step_timing in s: 2.899 | consumed_samples: 14112 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 441/999 | lr: 3.186e-06 | global_batch_size: 32 | global_step: 441 | reduced_train_loss: 0.1827 | train_step_timing in s: 2.902 | consumed_samples: 14144 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 442/999 | lr: 3.178e-06 | global_batch_size: 32 | global_step: 442 | reduced_train_loss: 0.2533 | train_step_timing in s: 2.9 | consumed_samples: 14176 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 443/999 | lr: 3.17e-06 | global_batch_size: 32 | global_step: 443 | reduced_train_loss: 0.2159 | train_step_timing in s: 2.902 | consumed_samples: 14208 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 444/999 | lr: 3.162e-06 | global_batch_size: 32 | global_step: 444 | reduced_train_loss: 0.2069 | train_step_timing in s: 2.903 | consumed_samples: 14240 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 445/999 | lr: 3.154e-06 | global_batch_size: 32 | global_step: 445 | reduced_train_loss: 0.2599 | train_step_timing in s: 2.897 | consumed_samples: 14272 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 446/999 | lr: 3.146e-06 | global_batch_size: 32 | global_step: 446 | reduced_train_loss: 0.2004 | train_step_timing in s: 2.9 | consumed_samples: 14304 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 447/999 | lr: 3.138e-06 | global_batch_size: 32 | global_step: 447 | reduced_train_loss: 0.2765 | train_step_timing in s: 2.901 | consumed_samples: 14336 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Training epoch 0, iteration 448/999 | lr: 3.13e-06 | global_batch_size: 32 | global_step: 448 | reduced_train_loss: 0.1825 | train_step_timing in s: 2.911 | consumed_samples: 14368 | val_loss: 0.2479
s_act_ckpt/0 [default0]:Epoch 0, global step 449: 'val_loss' reached 0.24793 (best 0.24793), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=449-consumed_samples=14400.0.ckpt' as top 2
s_act_ckpt/0 [default0]:Training epoch 0, iteration 449/999 | lr: 3.122e-06 | global_batch_size: 32 | global_step: 449 | reduced_train_loss: 0.2335 | train_step_timing in s: 2.897 | consumed_samples: 14400 | val_loss: 0.2479
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:33 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 449 : Start time: 1764083432.826s : Save duration: 0.183s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:33 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=449-consumed_samples=14400.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:10:33 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:Training epoch 0, iteration 450/999 | lr: 3.114e-06 | global_batch_size: 32 | global_step: 450 | reduced_train_loss: 0.2155 | train_step_timing in s: 2.904 | consumed_samples: 14432 | val_loss: 0.2462
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:11:03 pl:222] Successfully saved checkpoint from iteration     449 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=449-consumed_samples=14400.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:11:03 model_checkpoint:657] Async checkpoint save for step 450 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.25-step=449-consumed_samples=14400.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:11:05 dist_ckpt_io:160] Async finalization time took 1.359 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 451/999 | lr: 3.106e-06 | global_batch_size: 32 | global_step: 451 | reduced_train_loss: 0.2181 | train_step_timing in s: 2.9 | consumed_samples: 14464 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 452/999 | lr: 3.098e-06 | global_batch_size: 32 | global_step: 452 | reduced_train_loss: 0.2255 | train_step_timing in s: 2.892 | consumed_samples: 14496 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 453/999 | lr: 3.09e-06 | global_batch_size: 32 | global_step: 453 | reduced_train_loss: 0.3078 | train_step_timing in s: 2.897 | consumed_samples: 14528 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 454/999 | lr: 3.082e-06 | global_batch_size: 32 | global_step: 454 | reduced_train_loss: 0.4657 | train_step_timing in s: 2.905 | consumed_samples: 14560 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 455/999 | lr: 3.074e-06 | global_batch_size: 32 | global_step: 455 | reduced_train_loss: 0.2207 | train_step_timing in s: 2.904 | consumed_samples: 14592 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 456/999 | lr: 3.066e-06 | global_batch_size: 32 | global_step: 456 | reduced_train_loss: 0.2664 | train_step_timing in s: 2.899 | consumed_samples: 14624 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 457/999 | lr: 3.057e-06 | global_batch_size: 32 | global_step: 457 | reduced_train_loss: 0.3326 | train_step_timing in s: 2.896 | consumed_samples: 14656 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 458/999 | lr: 3.049e-06 | global_batch_size: 32 | global_step: 458 | reduced_train_loss: 0.1997 | train_step_timing in s: 2.907 | consumed_samples: 14688 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 459/999 | lr: 3.041e-06 | global_batch_size: 32 | global_step: 459 | reduced_train_loss: 0.221 | train_step_timing in s: 2.903 | consumed_samples: 14720 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 460/999 | lr: 3.033e-06 | global_batch_size: 32 | global_step: 460 | reduced_train_loss: 0.2284 | train_step_timing in s: 2.894 | consumed_samples: 14752 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 461/999 | lr: 3.025e-06 | global_batch_size: 32 | global_step: 461 | reduced_train_loss: 0.231 | train_step_timing in s: 2.901 | consumed_samples: 14784 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 462/999 | lr: 3.017e-06 | global_batch_size: 32 | global_step: 462 | reduced_train_loss: 0.2232 | train_step_timing in s: 2.897 | consumed_samples: 14816 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 463/999 | lr: 3.009e-06 | global_batch_size: 32 | global_step: 463 | reduced_train_loss: 0.2941 | train_step_timing in s: 2.899 | consumed_samples: 14848 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 464/999 | lr: 3.001e-06 | global_batch_size: 32 | global_step: 464 | reduced_train_loss: 0.255 | train_step_timing in s: 2.901 | consumed_samples: 14880 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 465/999 | lr: 2.993e-06 | global_batch_size: 32 | global_step: 465 | reduced_train_loss: 0.1841 | train_step_timing in s: 2.893 | consumed_samples: 14912 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 466/999 | lr: 2.985e-06 | global_batch_size: 32 | global_step: 466 | reduced_train_loss: 0.2564 | train_step_timing in s: 2.897 | consumed_samples: 14944 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 467/999 | lr: 2.977e-06 | global_batch_size: 32 | global_step: 467 | reduced_train_loss: 0.2669 | train_step_timing in s: 2.902 | consumed_samples: 14976 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 468/999 | lr: 2.968e-06 | global_batch_size: 32 | global_step: 468 | reduced_train_loss: 0.1829 | train_step_timing in s: 2.898 | consumed_samples: 15008 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 469/999 | lr: 2.96e-06 | global_batch_size: 32 | global_step: 469 | reduced_train_loss: 0.2696 | train_step_timing in s: 2.899 | consumed_samples: 15040 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 470/999 | lr: 2.952e-06 | global_batch_size: 32 | global_step: 470 | reduced_train_loss: 0.1727 | train_step_timing in s: 2.903 | consumed_samples: 15072 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 471/999 | lr: 2.944e-06 | global_batch_size: 32 | global_step: 471 | reduced_train_loss: 0.2357 | train_step_timing in s: 2.897 | consumed_samples: 15104 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 472/999 | lr: 2.936e-06 | global_batch_size: 32 | global_step: 472 | reduced_train_loss: 0.2412 | train_step_timing in s: 2.895 | consumed_samples: 15136 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 473/999 | lr: 2.928e-06 | global_batch_size: 32 | global_step: 473 | reduced_train_loss: 0.1922 | train_step_timing in s: 2.895 | consumed_samples: 15168 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 474/999 | lr: 2.92e-06 | global_batch_size: 32 | global_step: 474 | reduced_train_loss: 0.2214 | train_step_timing in s: 2.902 | consumed_samples: 15200 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 475/999 | lr: 2.911e-06 | global_batch_size: 32 | global_step: 475 | reduced_train_loss: 0.3078 | train_step_timing in s: 2.886 | consumed_samples: 15232 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 476/999 | lr: 2.903e-06 | global_batch_size: 32 | global_step: 476 | reduced_train_loss: 0.1995 | train_step_timing in s: 2.9 | consumed_samples: 15264 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 477/999 | lr: 2.895e-06 | global_batch_size: 32 | global_step: 477 | reduced_train_loss: 0.1761 | train_step_timing in s: 2.9 | consumed_samples: 15296 | val_loss: 0.2462[default0]:
s_act_ckpt/0 [default0]:Training epoch 0, iteration 478/999 | lr: 2.887e-06 | global_batch_size: 32 | global_step: 478 | reduced_train_loss: 0.1897 | train_step_timing in s: 2.906 | consumed_samples: 15328 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Training epoch 0, iteration 479/999 | lr: 2.879e-06 | global_batch_size: 32 | global_step: 479 | reduced_train_loss: 0.2983 | train_step_timing in s: 2.904 | consumed_samples: 15360 | val_loss: 0.2462
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:12:57 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 480 : Start time: 1764083576.982s : Save duration: 0.184s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:12:57 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.24-step=479-consumed_samples=15360.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 480/999 | lr: 2.871e-06 | global_batch_size: 32 | global_step: 480 | reduced_train_loss: 0.454 | train_step_timing in s: 2.905 | consumed_samples: 15392 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:00 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 481/999 | lr: 2.862e-06 | global_batch_size: 32 | global_step: 481 | reduced_train_loss: 0.2704 | train_step_timing in s: 2.9 | consumed_samples: 15424 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:03 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 482/999 | lr: 2.854e-06 | global_batch_size: 32 | global_step: 482 | reduced_train_loss: 0.2436 | train_step_timing in s: 2.904 | consumed_samples: 15456 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:06 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 483/999 | lr: 2.846e-06 | global_batch_size: 32 | global_step: 483 | reduced_train_loss: 0.3958 | train_step_timing in s: 2.897 | consumed_samples: 15488 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:09 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 484/999 | lr: 2.838e-06 | global_batch_size: 32 | global_step: 484 | reduced_train_loss: 0.1794 | train_step_timing in s: 2.896 | consumed_samples: 15520 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:12 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 485/999 | lr: 2.83e-06 | global_batch_size: 32 | global_step: 485 | reduced_train_loss: 0.3074 | train_step_timing in s: 2.899 | consumed_samples: 15552 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:15 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 486/999 | lr: 2.822e-06 | global_batch_size: 32 | global_step: 486 | reduced_train_loss: 0.2788 | train_step_timing in s: 2.894 | consumed_samples: 15584 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:18 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 487/999 | lr: 2.813e-06 | global_batch_size: 32 | global_step: 487 | reduced_train_loss: 0.3163 | train_step_timing in s: 2.898 | consumed_samples: 15616 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:21 pl:222] Successfully saved checkpoint from iteration     480 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.24-step=479-consumed_samples=15360.0-last.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:21 model_checkpoint:657] Async checkpoint save for step 480 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.24-step=479-consumed_samples=15360.0-last.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:21 dist_ckpt_io:160] Async finalization time took 0.057 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 488/999 | lr: 2.805e-06 | global_batch_size: 32 | global_step: 488 | reduced_train_loss: 0.2329 | train_step_timing in s: 2.902 | consumed_samples: 15648 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 489/999 | lr: 2.797e-06 | global_batch_size: 32 | global_step: 489 | reduced_train_loss: 0.3748 | train_step_timing in s: 2.905 | consumed_samples: 15680 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 490/999 | lr: 2.789e-06 | global_batch_size: 32 | global_step: 490 | reduced_train_loss: 0.2241 | train_step_timing in s: 2.897 | consumed_samples: 15712 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 491/999 | lr: 2.78e-06 | global_batch_size: 32 | global_step: 491 | reduced_train_loss: 0.1826 | train_step_timing in s: 2.897 | consumed_samples: 15744 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 492/999 | lr: 2.772e-06 | global_batch_size: 32 | global_step: 492 | reduced_train_loss: 0.2007 | train_step_timing in s: 2.904 | consumed_samples: 15776 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 493/999 | lr: 2.764e-06 | global_batch_size: 32 | global_step: 493 | reduced_train_loss: 0.3032 | train_step_timing in s: 2.894 | consumed_samples: 15808 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 494/999 | lr: 2.756e-06 | global_batch_size: 32 | global_step: 494 | reduced_train_loss: 0.1692 | train_step_timing in s: 2.891 | consumed_samples: 15840 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 495/999 | lr: 2.748e-06 | global_batch_size: 32 | global_step: 495 | reduced_train_loss: 0.2689 | train_step_timing in s: 2.901 | consumed_samples: 15872 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 496/999 | lr: 2.739e-06 | global_batch_size: 32 | global_step: 496 | reduced_train_loss: 0.2355 | train_step_timing in s: 2.9 | consumed_samples: 15904 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 497/999 | lr: 2.731e-06 | global_batch_size: 32 | global_step: 497 | reduced_train_loss: 0.1702 | train_step_timing in s: 2.893 | consumed_samples: 15936 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 498/999 | lr: 2.723e-06 | global_batch_size: 32 | global_step: 498 | reduced_train_loss: 0.2336 | train_step_timing in s: 2.896 | consumed_samples: 15968 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 499/999 | lr: 2.715e-06 | global_batch_size: 32 | global_step: 499 | reduced_train_loss: 0.218 | train_step_timing in s: 2.9 | consumed_samples: 16000 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Epoch 0, global step 499: 'val_loss' reached 0.24447 (best 0.24447), saving model to '/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.24-step=499-consumed_samples=16000.0.ckpt' as top 2
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:56 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 499 : Start time: 1764083636.214s : Save duration: 0.628s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:57 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.24-step=499-consumed_samples=16000.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:13:57 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 500/999 | lr: 2.706e-06 | global_batch_size: 32 | global_step: 500 | reduced_train_loss: 0.1868 | train_step_timing in s: 2.897 | consumed_samples: 16032 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:00 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 501/999 | lr: 2.698e-06 | global_batch_size: 32 | global_step: 501 | reduced_train_loss: 0.1924 | train_step_timing in s: 2.891 | consumed_samples: 16064 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:03 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 502/999 | lr: 2.69e-06 | global_batch_size: 32 | global_step: 502 | reduced_train_loss: 0.2068 | train_step_timing in s: 2.901 | consumed_samples: 16096 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:06 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 503/999 | lr: 2.682e-06 | global_batch_size: 32 | global_step: 503 | reduced_train_loss: 0.2346 | train_step_timing in s: 2.901 | consumed_samples: 16128 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:09 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 504/999 | lr: 2.673e-06 | global_batch_size: 32 | global_step: 504 | reduced_train_loss: 0.2075 | train_step_timing in s: 2.9 | consumed_samples: 16160 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:12 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 505/999 | lr: 2.665e-06 | global_batch_size: 32 | global_step: 505 | reduced_train_loss: 0.2996 | train_step_timing in s: 2.894 | consumed_samples: 16192 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:14 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 506/999 | lr: 2.657e-06 | global_batch_size: 32 | global_step: 506 | reduced_train_loss: 0.1974 | train_step_timing in s: 2.898 | consumed_samples: 16224 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:17 dist_ckpt_io:160] Async finalization time took 0.000 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 507/999 | lr: 2.649e-06 | global_batch_size: 32 | global_step: 507 | reduced_train_loss: 0.2291 | train_step_timing in s: 2.894 | consumed_samples: 16256 | val_loss: 0.2445
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:20 pl:222] Successfully saved checkpoint from iteration     499 to /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.24-step=499-consumed_samples=16000.0.ckpt
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:20 model_checkpoint:657] Async checkpoint save for step 500 (/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.24-step=499-consumed_samples=16000.0.ckpt) finalized successfully.
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:22 dist_ckpt_io:160] Async finalization time took 1.334 s
s_act_ckpt/0 [default0]:Training epoch 0, iteration 508/999 | lr: 2.64e-06 | global_batch_size: 32 | global_step: 508 | reduced_train_loss: 0.2315 | train_step_timing in s: 2.886 | consumed_samples: 16288 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Training epoch 0, iteration 509/999 | lr: 2.632e-06 | global_batch_size: 32 | global_step: 509 | reduced_train_loss: 0.2152 | train_step_timing in s: 2.895 | consumed_samples: 16320 | val_loss: 0.2445
s_act_ckpt/0 [default0]:Validation: iteration 1/0
s_act_ckpt/0 [default0]:Validation: iteration 2/0
s_act_ckpt/0 [default0]:Validation: iteration 3/0
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:54 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 510 : Start time: 1764083694.548s : Save duration: 0.180s
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:55 model_checkpoint:632] Scheduled async checkpoint save for /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/qwen3_600m_full_4k_tokens_act_ckpt/2025-11-25_15-41-07/checkpoints/model_name=0--val_loss=0.24-step=509-consumed_samples=16320.0-last.ckpt
s_act_ckpt/0 [default0]:Training epoch 0, iteration 510/999 | lr: 2.624e-06 | global_batch_size: 32 | global_step: 510 | reduced_train_loss: 0.1712 | train_step_timing in s: 2.9 | consumed_samples: 16352 | val_loss: 0.2416
s_act_ckpt/0 [default0]:[NeMo I 2025-11-25 16:14:58 dist_ckpt_io:160] Async finalization time took 0.000 s