[tab] /p/project1/envcomp/mihir/logs/eval/eval_miles_run/eval_miles_run_batch_13062811.log (log)
Extracted from RUN_NAME: RUN_ID=13032816, BASE_MODEL=Qwen3-8B
LoRA adapters found. Evaluating accordingly.

==============================================
PHASE 1: Generating outputs for all checkpoints
==============================================
Generating for base model (step 0)...
[2026-01-05 19:07:33] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:07:33] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:07:33] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:07:39] INFO server_args.py:1565: Attention backend not specified. Use flashinfer backend by default.
[2026-01-05 19:07:39] server_args=ServerArgs(model_path='/p/scratch/envcomp/mihir/huggingface/Qwen3-8B', tokenizer_path='/p/scratch/envcomp/mihir/huggingface/Qwen3-8B', tokenizer_mode='auto', tokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', trust_remote_code=False, context_length=15000, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', host='0.0.0.0', port=30000, fastapi_root_path='', grpc_mode=False, skip_server_warmup=False, warmups=None, nccl_port=None, checkpoint_engine_wait_weights_before_ready=False, dtype='auto', quantization=None, quantization_param_path=None, kv_cache_dtype='auto', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, rl_quant_profile=None, mem_fraction_static=0.85, max_running_requests=None, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=4096, enable_dynamic_chunking=False, max_prefill_tokens=16384, prefill_max_requests=None, schedule_policy='fcfs', enable_priority_scheduling=False, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, schedule_conservativeness=1.0, page_size=1, hybrid_kvcache_ratio=None, swa_full_tokens_ratio=0.8, disable_hybrid_swa_memory=False, radix_eviction_policy='lru', device='cuda', tp_size=4, pp_size=1, pp_max_micro_batch_size=None, pp_async_batch_depth=0, stream_interval=1, stream_output=False, random_seed=78433094, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, watchdog_timeout=300, soft_watchdog_timeout=None, dist_timeout=None, download_dir=None, base_gpu_id=0, gpu_id_step=1, sleep_on_idle=False, custom_sigquit_handler=None, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, log_requests_format='text', crash_dump_folder=None, show_time_cost=False, enable_metrics=False, enable_metrics_for_all_schedulers=False, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, collect_tokens_histogram=False, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_trace=False, otlp_traces_endpoint='localhost:4317', export_metrics_to_file=False, export_metrics_to_file_dir=None, api_key=None, served_model_name='/p/scratch/envcomp/mihir/huggingface/Qwen3-8B', weight_version='default', chat_template=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser=None, tool_call_parser=None, tool_server=None, sampling_defaults='model', dp_size=1, load_balance_method='round_robin', prefill_round_robin_balance=False, dist_init_addr=None, nnodes=1, node_rank=0, json_model_override_args='{}', preferred_sampling_params=None, enable_lora=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, attention_backend='flashinfer', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='flashinfer', grammar_backend='xgrammar', mm_attention_backend=None, fp8_gemm_runner_backend='auto', nsa_prefill_backend='flashmla_sparse', nsa_decode_backend='fa3', disable_flashinfer_autotune=False, speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_token_map=None, speculative_attention_mode='prefill', speculative_draft_attention_backend=None, speculative_moe_runner_backend='auto', speculative_moe_a2a_backend=None, speculative_draft_model_quantization=None, speculative_ngram_min_match_window_size=1, speculative_ngram_max_match_window_size=12, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_branch_length=18, speculative_ngram_capacity=10000000, enable_multi_layer_eagle=False, ep_size=1, moe_a2a_backend='none', moe_runner_backend='auto', flashinfer_mxfp4_moe_precision='default', enable_flashinfer_allreduce_fusion=False, deepep_mode='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm=None, init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, enable_expert_distribution_metrics=False, deepep_config=None, moe_dense_tp_size=None, elastic_ep_backend=None, mooncake_ib_device=None, max_mamba_cache_size=None, mamba_ssm_dtype='float32', mamba_full_memory_ratio=0.9, mamba_scheduler_strategy='no_buffer', mamba_track_interval=256, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='layer_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='best_effort', hicache_storage_backend_extra_config=None, enable_lmcache=False, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, dllm_algorithm=None, dllm_algorithm_config=None, enable_double_sparsity=False, ds_channel_config_path=None, ds_heavy_channel_num=32, ds_heavy_token_num=256, ds_heavy_channel_type='qk', ds_sparse_decode_threshold=4096, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', multi_item_scoring_delimiter=None, disable_radix_cache=False, cuda_graph_max_bs=160, cuda_graph_bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160], disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, enable_layerwise_nvtx_marker=False, enable_nccl_nvls=False, enable_symm_mem=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, disable_outlines_disk_cache=False, disable_custom_all_reduce=False, enable_mscclpp=False, enable_torch_symm_mem=False, disable_overlap_schedule=False, enable_mixed_chunk=False, enable_dp_attention=False, enable_dp_lm_head=False, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, enable_torch_compile=False, enable_piecewise_cuda_graph=False, enable_torch_compile_debug_mode=False, torch_compile_max_bs=32, piecewise_cuda_graph_max_tokens=4096, piecewise_cuda_graph_tokens=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 640, 768, 896, 1024, 1152, 1280, 1408, 1536, 1664, 1792, 1920, 2048, 2176, 2304, 2432, 2560, 2688, 2816, 2944, 3072, 3200, 3328, 3456, 3584, 3712, 3840, 3968, 4096], piecewise_cuda_graph_compiler='eager', torchao_config='', enable_nan_detection=False, enable_p2p_check=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, num_continuous_decode_steps=1, delete_ckpt_after_loading=False, enable_memory_saver=False, enable_weights_cpu_backup=False, enable_draft_weights_cpu_backup=False, allow_auto_truncate=False, enable_custom_logit_processor=False, flashinfer_mla_disable_ragged=False, disable_shared_experts_fusion=False, disable_chunked_prefix_cache=False, disable_fast_image_processor=False, keep_mm_feature_on_device=False, enable_return_hidden_states=False, enable_return_routed_experts=False, scheduler_recv_interval=1, numa_node=None, enable_deterministic_inference=False, rl_on_policy_target=None, enable_attn_tp_input_scattered=False, enable_nsa_prefill_context_parallel=False, enable_fused_qk_norm_rope=False, enable_precise_embedding_interpolation=False, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, debug_tensor_dump_inject=False, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_decode_tp=None, disaggregation_decode_dp=None, disaggregation_prefill_pp=1, disaggregation_ib_device=None, disaggregation_decode_enable_offload_kvcache=False, disaggregation_decode_enable_fake_auto=False, num_reserved_decode_tokens=512, disaggregation_decode_polling_interval=1, encoder_only=False, language_only=False, encoder_transfer_backend='zmq_to_scheduler', encoder_urls=[], custom_weight_loader=[], weight_loader_disable_mmap=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, remote_instance_weight_loader_backend='nccl', remote_instance_weight_loader_start_seed_via_transfer_engine=False, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, mm_max_concurrent_calls=32, mm_per_request_timeout=10.0, enable_broadcast_mm_inputs_process=False, enable_prefix_mm_cache=False, mm_enable_dp_encoder=False, mm_process_config={}, limit_mm_data_per_request=None, decrypted_config_file=None, decrypted_draft_config_file=None, forward_hooks=None)
[2026-01-05 19:07:40] Using default HuggingFace chat template with detected content format: string
[2026-01-05 19:07:48] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:07:48] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:07:48] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:07:48] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:07:48] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:07:48] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:07:49] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:07:49] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:07:49] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:07:49] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:07:49] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:07:49] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:07:49] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:07:49] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:07:49] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:07:51 TP1] Init torch distributed begin.
[W105 19:07:52.150664127 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [localhost.localdomain]:30442 (errno: 97 - Address family not supported by protocol).
[2026-01-05 19:07:52 TP2] Init torch distributed begin.
[2026-01-05 19:07:52 TP0] Init torch distributed begin.
[2026-01-05 19:07:52 TP3] Init torch distributed begin.
[W105 19:07:53.378902305 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [localhost.localdomain]:30442 (errno: 97 - Address family not supported by protocol).
[W105 19:07:53.384901035 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [localhost.localdomain]:30442 (errno: 97 - Address family not supported by protocol).
[W105 19:07:53.396628694 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [localhost.localdomain]:30442 (errno: 97 - Address family not supported by protocol).
[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[2026-01-05 19:07:54 TP0] sglang is using nccl==2.27.5
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[2026-01-05 19:07:55 TP0] Init torch distributed ends. mem usage=1.01 GB
[2026-01-05 19:07:55 TP1] Init torch distributed ends. mem usage=1.08 GB
[2026-01-05 19:07:55 TP2] Init torch distributed ends. mem usage=1.01 GB
[2026-01-05 19:07:55 TP3] Init torch distributed ends. mem usage=1.08 GB
[2026-01-05 19:07:55 TP0] MOE_RUNNER_BACKEND is not initialized, the backend will be automatically selected
[2026-01-05 19:07:59 TP0] Ignore import error when loading sglang.srt.models.glmasr: cannot import name 'GlmAsrConfig' from 'transformers' (/p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/transformers/__init__.py)
[2026-01-05 19:07:59 TP2] Ignore import error when loading sglang.srt.models.glmasr: cannot import name 'GlmAsrConfig' from 'transformers' (/p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/transformers/__init__.py)
[2026-01-05 19:07:59 TP1] Ignore import error when loading sglang.srt.models.glmasr: cannot import name 'GlmAsrConfig' from 'transformers' (/p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/transformers/__init__.py)
[2026-01-05 19:07:59 TP3] Ignore import error when loading sglang.srt.models.glmasr: cannot import name 'GlmAsrConfig' from 'transformers' (/p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/transformers/__init__.py)
[2026-01-05 19:07:59 TP1] Load weight begin. avail mem=37.99 GB
[2026-01-05 19:07:59 TP2] Load weight begin. avail mem=38.06 GB
[2026-01-05 19:07:59 TP0] Load weight begin. avail mem=38.06 GB
[2026-01-05 19:07:59 TP3] Load weight begin. avail mem=37.99 GB

Loading safetensors checkpoint shards:   0% Completed | 0/5 [00:00<?, ?it/s]

Loading safetensors checkpoint shards:  20% Completed | 1/5 [00:00<00:03,  1.12it/s]

Loading safetensors checkpoint shards:  40% Completed | 2/5 [00:05<00:08,  2.88s/it]

Loading safetensors checkpoint shards:  60% Completed | 3/5 [00:09<00:06,  3.45s/it]

Loading safetensors checkpoint shards:  80% Completed | 4/5 [00:14<00:04,  4.08s/it]

Loading safetensors checkpoint shards: 100% Completed | 5/5 [00:21<00:00,  5.07s/it]

Loading safetensors checkpoint shards: 100% Completed | 5/5 [00:21<00:00,  4.23s/it]

[2026-01-05 19:08:21 TP0] Load weight end. type=Qwen3ForCausalLM, dtype=torch.bfloat16, avail mem=34.22 GB, mem usage=3.84 GB.
[2026-01-05 19:08:21 TP1] Load weight end. type=Qwen3ForCausalLM, dtype=torch.bfloat16, avail mem=34.15 GB, mem usage=3.84 GB.
[2026-01-05 19:08:21 TP2] Load weight end. type=Qwen3ForCausalLM, dtype=torch.bfloat16, avail mem=34.22 GB, mem usage=3.84 GB.
[2026-01-05 19:08:21 TP3] Load weight end. type=Qwen3ForCausalLM, dtype=torch.bfloat16, avail mem=34.15 GB, mem usage=3.84 GB.
[2026-01-05 19:08:21 TP0] Using KV cache dtype: torch.bfloat16
[2026-01-05 19:08:21 TP0] KV Cache is allocated. #tokens: 828711, K size: 14.23 GB, V size: 14.23 GB
[2026-01-05 19:08:21 TP1] KV Cache is allocated. #tokens: 828711, K size: 14.23 GB, V size: 14.23 GB
[2026-01-05 19:08:21 TP2] KV Cache is allocated. #tokens: 828711, K size: 14.23 GB, V size: 14.23 GB
[2026-01-05 19:08:21 TP3] KV Cache is allocated. #tokens: 828711, K size: 14.23 GB, V size: 14.23 GB
[2026-01-05 19:08:21 TP0] Memory pool end. avail mem=5.44 GB
[2026-01-05 19:08:21 TP2] Memory pool end. avail mem=5.44 GB
[2026-01-05 19:08:21 TP1] Memory pool end. avail mem=5.37 GB
[2026-01-05 19:08:21 TP3] Memory pool end. avail mem=5.37 GB
[2026-01-05 19:08:21 TP0] Capture cuda graph begin. This can take up to several minutes. avail mem=4.81 GB
[2026-01-05 19:08:21 TP3] Capture cuda graph begin. This can take up to several minutes. avail mem=4.74 GB
[2026-01-05 19:08:21 TP1] Capture cuda graph begin. This can take up to several minutes. avail mem=4.74 GB
[2026-01-05 19:08:21 TP0] Capture cuda graph bs [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160]
[2026-01-05 19:08:21 TP2] Capture cuda graph begin. This can take up to several minutes. avail mem=4.81 GB

  0%|          | 0/24 [00:00<?, ?it/s]
Capturing batches (bs=160 avail_mem=4.71 GB):   0%|          | 0/24 [00:00<?, ?it/s]
Capturing batches (bs=160 avail_mem=4.71 GB):   4%|▍         | 1/24 [00:06<02:32,  6.62s/it]
Capturing batches (bs=152 avail_mem=4.56 GB):   4%|▍         | 1/24 [00:06<02:32,  6.62s/it]
Capturing batches (bs=152 avail_mem=4.56 GB):   8%|▊         | 2/24 [00:06<01:04,  2.94s/it]
Capturing batches (bs=144 avail_mem=4.54 GB):   8%|▊         | 2/24 [00:06<01:04,  2.94s/it]
Capturing batches (bs=136 avail_mem=4.54 GB):   8%|▊         | 2/24 [00:07<01:04,  2.94s/it]
Capturing batches (bs=136 avail_mem=4.54 GB):  17%|█▋        | 4/24 [00:07<00:22,  1.15s/it]
Capturing batches (bs=128 avail_mem=4.51 GB):  17%|█▋        | 4/24 [00:07<00:22,  1.15s/it]
Capturing batches (bs=128 avail_mem=4.51 GB):  21%|██        | 5/24 [00:07<00:15,  1.21it/s]
Capturing batches (bs=120 avail_mem=4.50 GB):  21%|██        | 5/24 [00:07<00:15,  1.21it/s]
Capturing batches (bs=112 avail_mem=4.47 GB):  21%|██        | 5/24 [00:07<00:15,  1.21it/s]
Capturing batches (bs=112 avail_mem=4.47 GB):  29%|██▉       | 7/24 [00:07<00:08,  2.08it/s]
Capturing batches (bs=104 avail_mem=4.46 GB):  29%|██▉       | 7/24 [00:07<00:08,  2.08it/s]
Capturing batches (bs=96 avail_mem=4.43 GB):  29%|██▉       | 7/24 [00:07<00:08,  2.08it/s] 
Capturing batches (bs=96 avail_mem=4.43 GB):  38%|███▊      | 9/24 [00:07<00:04,  3.09it/s]
Capturing batches (bs=88 avail_mem=4.42 GB):  38%|███▊      | 9/24 [00:07<00:04,  3.09it/s]
Capturing batches (bs=80 avail_mem=4.39 GB):  38%|███▊      | 9/24 [00:07<00:04,  3.09it/s]
Capturing batches (bs=80 avail_mem=4.39 GB):  46%|████▌     | 11/24 [00:07<00:03,  4.20it/s]
Capturing batches (bs=72 avail_mem=4.38 GB):  46%|████▌     | 11/24 [00:07<00:03,  4.20it/s]
Capturing batches (bs=64 avail_mem=4.35 GB):  46%|████▌     | 11/24 [00:07<00:03,  4.20it/s]
Capturing batches (bs=64 avail_mem=4.35 GB):  54%|█████▍    | 13/24 [00:07<00:02,  5.29it/s]
Capturing batches (bs=56 avail_mem=4.34 GB):  54%|█████▍    | 13/24 [00:07<00:02,  5.29it/s]
Capturing batches (bs=48 avail_mem=4.32 GB):  54%|█████▍    | 13/24 [00:08<00:02,  5.29it/s]
Capturing batches (bs=48 avail_mem=4.32 GB):  62%|██████▎   | 15/24 [00:08<00:01,  6.42it/s]
Capturing batches (bs=40 avail_mem=4.31 GB):  62%|██████▎   | 15/24 [00:08<00:01,  6.42it/s]
Capturing batches (bs=32 avail_mem=4.29 GB):  62%|██████▎   | 15/24 [00:08<00:01,  6.42it/s]
Capturing batches (bs=32 avail_mem=4.29 GB):  71%|███████   | 17/24 [00:08<00:00,  7.31it/s]
Capturing batches (bs=24 avail_mem=4.28 GB):  71%|███████   | 17/24 [00:08<00:00,  7.31it/s]
Capturing batches (bs=16 avail_mem=4.26 GB):  71%|███████   | 17/24 [00:08<00:00,  7.31it/s]
Capturing batches (bs=16 avail_mem=4.26 GB):  79%|███████▉  | 19/24 [00:08<00:00,  8.25it/s]
Capturing batches (bs=12 avail_mem=4.25 GB):  79%|███████▉  | 19/24 [00:08<00:00,  8.25it/s]
Capturing batches (bs=8 avail_mem=4.22 GB):  79%|███████▉  | 19/24 [00:08<00:00,  8.25it/s] 
Capturing batches (bs=8 avail_mem=4.22 GB):  88%|████████▊ | 21/24 [00:08<00:00,  9.08it/s]
Capturing batches (bs=4 avail_mem=4.22 GB):  88%|████████▊ | 21/24 [00:08<00:00,  9.08it/s]
Capturing batches (bs=2 avail_mem=4.19 GB):  88%|████████▊ | 21/24 [00:08<00:00,  9.08it/s]
Capturing batches (bs=2 avail_mem=4.19 GB):  96%|█████████▌| 23/24 [00:08<00:00,  9.71it/s]
Capturing batches (bs=1 avail_mem=4.18 GB):  96%|█████████▌| 23/24 [00:08<00:00,  9.71it/s]
Capturing batches (bs=1 avail_mem=4.18 GB): 100%|██████████| 24/24 [00:09<00:00,  2.59it/s]
[2026-01-05 19:08:31 TP0] Registering 1752 cuda graph addresses
[2026-01-05 19:08:31 TP3] Capture cuda graph end. Time elapsed: 10.09 s. mem usage=0.65 GB. avail mem=4.09 GB.
[2026-01-05 19:08:31 TP0] Capture cuda graph end. Time elapsed: 10.12 s. mem usage=0.65 GB. avail mem=4.16 GB.
[2026-01-05 19:08:31 TP2] Capture cuda graph end. Time elapsed: 10.13 s. mem usage=0.65 GB. avail mem=4.16 GB.
[2026-01-05 19:08:31 TP1] Capture cuda graph end. Time elapsed: 10.14 s. mem usage=0.65 GB. avail mem=4.09 GB.
[2026-01-05 19:08:32 TP0] max_total_num_tokens=828711, chunked_prefill_size=4096, max_prefill_tokens=16384, max_running_requests=4096, context_len=15000, available_gpu_mem=4.16 GB
[2026-01-05 19:08:32] INFO:     Started server process [3018102]
[2026-01-05 19:08:32] INFO:     Waiting for application startup.
[2026-01-05 19:08:32] Using default chat sampling params from model generation config: {'repetition_penalty': 1.0, 'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
[2026-01-05 19:08:32] Using default chat sampling params from model generation config: {'repetition_penalty': 1.0, 'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
[2026-01-05 19:08:32] INFO:     Application startup complete.
[2026-01-05 19:08:33] INFO:     Uvicorn running on http://0.0.0.0:30000 (Press CTRL+C to quit)
[2026-01-05 19:08:34] INFO:     127.0.0.1:60544 - "GET /model_info HTTP/1.1" 200 OK
[2026-01-05 19:08:34 TP0] Prefill batch, #new-seq: 1, #new-token: 6, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, 
[2026-01-05 19:08:35] INFO:     127.0.0.1:33480 - "GET /v1/models HTTP/1.1" 200 OK
Using server at http://0.0.0.0:30000/v1
Launching sglang server:
  /p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/bin/python -m sglang.launch_server --model-path /p/scratch/envcomp/mihir/huggingface/Qwen3-8B --host 0.0.0.0 --port 30000 --context-length 15000 --mem-fraction-static 0.85 --tp-size 4
Waiting for server at http://0.0.0.0:30000 ...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server is up.

Filtered dataset:
  Valid test cases: 5
  Skipped (too long): 0

Running 5 test cases with concurrency=4 ...

  0%|          | 0/5 [00:00<?, ?it/s][2026-01-05 19:08:36 TP0] Prefill batch, #new-seq: 1, #new-token: 4096, #cached-token: 0, token usage: 0.00, #running-req: 1, #queue-req: 3, 
[2026-01-05 19:08:36 TP0] Prefill batch, #new-seq: 3, #new-token: 4096, #cached-token: 4187, token usage: 0.00, #running-req: 1, #queue-req: 1, 
[2026-01-05 19:08:37 TP0] Prefill batch, #new-seq: 2, #new-token: 308, #cached-token: 888, token usage: 0.01, #running-req: 1, #queue-req: 0, 
[2026-01-05 19:08:37 TP0] Prefill batch, #new-seq: 40, #new-token: 40, #cached-token: 135710, token usage: 0.01, #running-req: 1, #queue-req: 0, 
[2026-01-05 19:08:38] INFO:     127.0.0.1:60556 - "POST /generate HTTP/1.1" 200 OK
[2026-01-05 19:08:38] The server is fired up and ready to roll!
[2026-01-05 19:08:38 TP0] Decode batch, #running-req: 40, #token: 8713, token usage: 0.01, cuda graph: True, gen throughput (token/s): 33.65, #queue-req: 0, 
[2026-01-05 19:08:38 TP0] Decode batch, #running-req: 40, #token: 10313, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4911.67, #queue-req: 0, 
[2026-01-05 19:08:39 TP0] Decode batch, #running-req: 40, #token: 11913, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4897.75, #queue-req: 0, 
[2026-01-05 19:08:39 TP0] Decode batch, #running-req: 40, #token: 13513, token usage: 0.02, cuda graph: True, gen throughput (token/s): 4887.12, #queue-req: 0, 
[2026-01-05 19:08:39 TP0] Decode batch, #running-req: 40, #token: 15113, token usage: 0.02, cuda graph: True, gen throughput (token/s): 4868.65, #queue-req: 0, 
[2026-01-05 19:08:40 TP0] Decode batch, #running-req: 39, #token: 16473, token usage: 0.02, cuda graph: True, gen throughput (token/s): 4829.76, #queue-req: 0, 
[2026-01-05 19:08:40 TP0] Decode batch, #running-req: 38, #token: 17473, token usage: 0.02, cuda graph: True, gen throughput (token/s): 4716.49, #queue-req: 0, 
[2026-01-05 19:08:40 TP0] Decode batch, #running-req: 32, #token: 17353, token usage: 0.02, cuda graph: True, gen throughput (token/s): 4186.88, #queue-req: 0, 
[2026-01-05 19:08:41 TP0] Decode batch, #running-req: 30, #token: 17553, token usage: 0.02, cuda graph: True, gen throughput (token/s): 3993.56, #queue-req: 0, 
[2026-01-05 19:08:41 TP0] Decode batch, #running-req: 29, #token: 18313, token usage: 0.02, cuda graph: True, gen throughput (token/s): 3662.91, #queue-req: 0, 
[2026-01-05 19:08:41 TP0] Decode batch, #running-req: 26, #token: 18113, token usage: 0.02, cuda graph: True, gen throughput (token/s): 3393.51, #queue-req: 0, 
[2026-01-05 19:08:42 TP0] Decode batch, #running-req: 22, #token: 17673, token usage: 0.02, cuda graph: True, gen throughput (token/s): 3121.49, #queue-req: 0, 
[2026-01-05 19:08:42 TP0] Decode batch, #running-req: 19, #token: 16993, token usage: 0.02, cuda graph: True, gen throughput (token/s): 2734.94, #queue-req: 0, 
[2026-01-05 19:08:42] INFO:     127.0.0.1:33528 - "POST /v1/chat/completions HTTP/1.1" 200 OK

 20%|██        | 1/5 [00:07<00:29,  7.34s/it][2026-01-05 19:08:42 TP0] Prefill batch, #new-seq: 1, #new-token: 4096, #cached-token: 91, token usage: 0.02, #running-req: 17, #queue-req: 0, 
[2026-01-05 19:08:42 TP0] Prefill batch, #new-seq: 1, #new-token: 2923, #cached-token: 0, token usage: 0.02, #running-req: 17, #queue-req: 0, 
[2026-01-05 19:08:42 TP0] Prefill batch, #new-seq: 10, #new-token: 10, #cached-token: 71090, token usage: 0.03, #running-req: 17, #queue-req: 0, 
[2026-01-05 19:08:43 TP0] Decode batch, #running-req: 26, #token: 23114, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1312.22, #queue-req: 0, 
[2026-01-05 19:08:43 TP0] Decode batch, #running-req: 24, #token: 22954, token usage: 0.03, cuda graph: True, gen throughput (token/s): 3175.16, #queue-req: 0, 
[2026-01-05 19:08:43 TP0] Decode batch, #running-req: 23, #token: 23274, token usage: 0.03, cuda graph: True, gen throughput (token/s): 3081.58, #queue-req: 0, 
[2026-01-05 19:08:44 TP0] Decode batch, #running-req: 23, #token: 24194, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2957.18, #queue-req: 0, 
[2026-01-05 19:08:44 TP0] Decode batch, #running-req: 23, #token: 25114, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2951.90, #queue-req: 0, 
[2026-01-05 19:08:44 TP0] Decode batch, #running-req: 21, #token: 24514, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2822.41, #queue-req: 0, 
[2026-01-05 19:08:45 TP0] Decode batch, #running-req: 20, #token: 24554, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2635.26, #queue-req: 0, 
[2026-01-05 19:08:45 TP0] Decode batch, #running-req: 20, #token: 25354, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2601.55, #queue-req: 0, 
[2026-01-05 19:08:45 TP0] Decode batch, #running-req: 19, #token: 25274, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2557.21, #queue-req: 0, 
[2026-01-05 19:08:45 TP0] Decode batch, #running-req: 19, #token: 26034, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2499.09, #queue-req: 0, 
[2026-01-05 19:08:46 TP0] Decode batch, #running-req: 18, #token: 25834, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2460.04, #queue-req: 0, 
[2026-01-05 19:08:46 TP0] Decode batch, #running-req: 18, #token: 26554, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2378.76, #queue-req: 0, 
[2026-01-05 19:08:46 TP0] Decode batch, #running-req: 18, #token: 27274, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2368.57, #queue-req: 0, 
[2026-01-05 19:08:47] INFO:     127.0.0.1:33512 - "POST /v1/chat/completions HTTP/1.1" 200 OK

 40%|████      | 2/5 [00:11<00:16,  5.64s/it][2026-01-05 19:08:47 TP0] Decode batch, #running-req: 16, #token: 25504, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2275.39, #queue-req: 0, 
[2026-01-05 19:08:47 TP0] Decode batch, #running-req: 14, #token: 24988, token usage: 0.03, cuda graph: True, gen throughput (token/s): 2006.07, #queue-req: 0, 
[2026-01-05 19:08:47 TP0] Decode batch, #running-req: 13, #token: 24930, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1965.44, #queue-req: 0, 
[2026-01-05 19:08:47 TP0] Decode batch, #running-req: 11, #token: 23592, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1671.46, #queue-req: 0, 
[2026-01-05 19:08:48 TP0] Decode batch, #running-req: 11, #token: 24032, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1599.09, #queue-req: 0, 
[2026-01-05 19:08:48 TP0] Decode batch, #running-req: 10, #token: 23734, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1497.07, #queue-req: 0, 
[2026-01-05 19:08:48 TP0] Decode batch, #running-req: 8, #token: 22036, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1356.38, #queue-req: 0, 
[2026-01-05 19:08:49 TP0] Decode batch, #running-req: 8, #token: 22356, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1211.58, #queue-req: 0, 
[2026-01-05 19:08:49 TP0] Decode batch, #running-req: 7, #token: 21276, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1106.47, #queue-req: 0, 
[2026-01-05 19:08:49 TP0] Decode batch, #running-req: 7, #token: 21556, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1076.45, #queue-req: 0, 
[2026-01-05 19:08:49 TP0] Decode batch, #running-req: 7, #token: 21836, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1073.32, #queue-req: 0, 
[2026-01-05 19:08:50 TP0] Decode batch, #running-req: 7, #token: 22116, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1072.40, #queue-req: 0, 
[2026-01-05 19:08:50 TP0] Decode batch, #running-req: 7, #token: 22396, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1070.08, #queue-req: 0, 
[2026-01-05 19:08:50 TP0] Decode batch, #running-req: 6, #token: 21076, token usage: 0.03, cuda graph: True, gen throughput (token/s): 1007.41, #queue-req: 0, 
[2026-01-05 19:08:50 TP0] Decode batch, #running-req: 5, #token: 19676, token usage: 0.02, cuda graph: True, gen throughput (token/s): 852.79, #queue-req: 0, 
[2026-01-05 19:08:51 TP0] Decode batch, #running-req: 5, #token: 19876, token usage: 0.02, cuda graph: True, gen throughput (token/s): 766.06, #queue-req: 0, 
[2026-01-05 19:08:51 TP0] Decode batch, #running-req: 5, #token: 20076, token usage: 0.02, cuda graph: True, gen throughput (token/s): 767.16, #queue-req: 0, 
[2026-01-05 19:08:51] INFO:     127.0.0.1:33490 - "POST /v1/chat/completions HTTP/1.1" 200 OK

 60%|██████    | 3/5 [00:16<00:10,  5.15s/it][2026-01-05 19:08:51 TP0] Decode batch, #running-req: 4, #token: 18516, token usage: 0.02, cuda graph: True, gen throughput (token/s): 741.35, #queue-req: 0, 
[2026-01-05 19:08:51 TP0] Decode batch, #running-req: 4, #token: 18676, token usage: 0.02, cuda graph: True, gen throughput (token/s): 636.47, #queue-req: 0, 
[2026-01-05 19:08:52 TP0] Decode batch, #running-req: 4, #token: 18836, token usage: 0.02, cuda graph: True, gen throughput (token/s): 637.53, #queue-req: 0, 
[2026-01-05 19:08:52 TP0] Decode batch, #running-req: 4, #token: 18996, token usage: 0.02, cuda graph: True, gen throughput (token/s): 637.47, #queue-req: 0, 
[2026-01-05 19:08:52 TP0] Decode batch, #running-req: 4, #token: 19156, token usage: 0.02, cuda graph: True, gen throughput (token/s): 636.64, #queue-req: 0, 
[2026-01-05 19:08:52 TP0] Decode batch, #running-req: 4, #token: 19316, token usage: 0.02, cuda graph: True, gen throughput (token/s): 638.10, #queue-req: 0, 
[2026-01-05 19:08:53 TP0] Decode batch, #running-req: 4, #token: 19476, token usage: 0.02, cuda graph: True, gen throughput (token/s): 637.29, #queue-req: 0, 
[2026-01-05 19:08:53 TP0] Decode batch, #running-req: 4, #token: 19636, token usage: 0.02, cuda graph: True, gen throughput (token/s): 636.01, #queue-req: 0, 
[2026-01-05 19:08:53 TP0] Decode batch, #running-req: 4, #token: 19796, token usage: 0.02, cuda graph: True, gen throughput (token/s): 636.49, #queue-req: 0, 
[2026-01-05 19:08:53 TP0] Decode batch, #running-req: 4, #token: 19956, token usage: 0.02, cuda graph: True, gen throughput (token/s): 635.75, #queue-req: 0, 
[2026-01-05 19:08:54 TP0] Decode batch, #running-req: 3, #token: 18498, token usage: 0.02, cuda graph: True, gen throughput (token/s): 495.67, #queue-req: 0, 
[2026-01-05 19:08:54 TP0] Decode batch, #running-req: 3, #token: 18618, token usage: 0.02, cuda graph: True, gen throughput (token/s): 483.21, #queue-req: 0, 
[2026-01-05 19:08:54 TP0] Decode batch, #running-req: 2, #token: 17040, token usage: 0.02, cuda graph: True, gen throughput (token/s): 472.34, #queue-req: 0, 
[2026-01-05 19:08:54] INFO:     127.0.0.1:33528 - "POST /v1/chat/completions HTTP/1.1" 200 OK

 80%|████████  | 4/5 [00:19<00:04,  4.33s/it][2026-01-05 19:08:54 TP0] Decode batch, #running-req: 1, #token: 8363, token usage: 0.01, cuda graph: True, gen throughput (token/s): 196.28, #queue-req: 0, 
[2026-01-05 19:08:55 TP0] Decode batch, #running-req: 1, #token: 8403, token usage: 0.01, cuda graph: True, gen throughput (token/s): 180.26, #queue-req: 0, 
[2026-01-05 19:08:55 TP0] Decode batch, #running-req: 1, #token: 8443, token usage: 0.01, cuda graph: True, gen throughput (token/s): 180.16, #queue-req: 0, 
[2026-01-05 19:08:55 TP0] Decode batch, #running-req: 1, #token: 8483, token usage: 0.01, cuda graph: True, gen throughput (token/s): 178.51, #queue-req: 0, 
[2026-01-05 19:08:55 TP0] Decode batch, #running-req: 1, #token: 8523, token usage: 0.01, cuda graph: True, gen throughput (token/s): 179.90, #queue-req: 0, 
[2026-01-05 19:08:56 TP0] Decode batch, #running-req: 1, #token: 8563, token usage: 0.01, cuda graph: True, gen throughput (token/s): 179.94, #queue-req: 0, 
[2026-01-05 19:08:56 TP0] Decode batch, #running-req: 1, #token: 8603, token usage: 0.01, cuda graph: True, gen throughput (token/s): 179.79, #queue-req: 0, 
[2026-01-05 19:08:56 TP0] Decode batch, #running-req: 1, #token: 8643, token usage: 0.01, cuda graph: True, gen throughput (token/s): 180.01, #queue-req: 0, 
[2026-01-05 19:08:56 TP0] Decode batch, #running-req: 1, #token: 8683, token usage: 0.01, cuda graph: True, gen throughput (token/s): 179.95, #queue-req: 0, 
[2026-01-05 19:08:56 TP0] Decode batch, #running-req: 1, #token: 8723, token usage: 0.01, cuda graph: True, gen throughput (token/s): 177.91, #queue-req: 0, 
[2026-01-05 19:08:57] INFO:     127.0.0.1:33504 - "POST /v1/chat/completions HTTP/1.1" 200 OK

100%|██████████| 5/5 [00:21<00:00,  3.60s/it]
100%|██████████| 5/5 [00:21<00:00,  4.35s/it]
[2026-01-05 19:08:57] SIGTERM received. signum=None frame=None. Draining requests and shutting down...
[2026-01-05 19:08:59] Gracefully exiting... Remaining number of requests 0. Remaining requests remaining_rids=[].

==================================================
--- Evaluation Complete ---
==================================================
Total Test Cases: 5
Number of Samples per Task: 10
Total Exact Match Pass At N: 3
Total Exact Match Avg At N: 38.00%
Generations output file: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_0_generations.jsonl
Shutting down sglang server ...
Done
LoRA adapter dir: /p/scratch/envcomp/mihir/huggingface/shared_data/13032816/checkpoints/iter_0000500/, Step: 500
Generating for step 500...
LoRA adapter dir: /p/scratch/envcomp/mihir/huggingface/shared_data/13032816/checkpoints/iter_0001000/, Step: 1000
Generating for step 1000...
LoRA adapter dir: /p/scratch/envcomp/mihir/huggingface/shared_data/13032816/checkpoints/iter_0001500/, Step: 1500
Generating for step 1500...
LoRA adapter dir: /p/scratch/envcomp/mihir/huggingface/shared_data/13032816/checkpoints/iter_0002000/, Step: 2000
Generating for step 2000...
LoRA adapter dir: /p/scratch/envcomp/mihir/huggingface/shared_data/13032816/checkpoints/iter_0002500/, Step: 2500
Generating for step 2500...
LoRA adapter dir: /p/scratch/envcomp/mihir/huggingface/shared_data/13032816/checkpoints/iter_0003000/, Step: 3000
Generating for step 3000...
LoRA adapter dir: /p/scratch/envcomp/mihir/huggingface/shared_data/13032816/checkpoints/iter_0003500/, Step: 3500
Generating for step 3500...
LoRA adapter dir: /p/scratch/envcomp/mihir/huggingface/shared_data/13032816/checkpoints/iter_0004000/, Step: 4000
Generating for step 4000...
LoRA adapter dir: /p/scratch/envcomp/mihir/huggingface/shared_data/13032816/checkpoints/iter_0004500/, Step: 4500
Generating for step 4500...
LoRA adapter dir: /p/scratch/envcomp/mihir/huggingface/shared_data/13032816/checkpoints/iter_0005000/, Step: 5000
Generating for step 5000...

==============================================
PHASE 2: Batch evaluation (single judge load)
==============================================
Running batch evaluation with:
  Generations files: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_0_generations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_500_generations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_1000_generations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_1500_generations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_2000_generations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_2500_generations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_3000_generations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_3500_generations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_4000_generations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_4500_generations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_5000_generations.jsonl
  Evaluations files: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_0_evaluations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_500_evaluations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_1000_evaluations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_1500_evaluations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_2000_evaluations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_2500_evaluations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_3000_evaluations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_3500_evaluations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_4000_evaluations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_4500_evaluations.jsonl,data/eval/output/13032816/qwen8B_lora_bs_32_13032816_5000_evaluations.jsonl
  Eval steps: 0,500,1000,1500,2000,2500,3000,3500,4000,4500,5000
[2026-01-05 19:09:14] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:09:14] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:09:14] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:09:18] INFO server_args.py:1565: Attention backend not specified. Use flashinfer backend by default.
[2026-01-05 19:09:19] server_args=ServerArgs(model_path='/p/scratch/envcomp/mihir/huggingface/Qwen3-30B-A3B', tokenizer_path='/p/scratch/envcomp/mihir/huggingface/Qwen3-30B-A3B', tokenizer_mode='auto', tokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', trust_remote_code=False, context_length=40960, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', host='localhost', port=30000, fastapi_root_path='', grpc_mode=False, skip_server_warmup=False, warmups=None, nccl_port=None, checkpoint_engine_wait_weights_before_ready=False, dtype='auto', quantization=None, quantization_param_path=None, kv_cache_dtype='auto', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, rl_quant_profile=None, mem_fraction_static=0.85, max_running_requests=None, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=4096, enable_dynamic_chunking=False, max_prefill_tokens=16384, prefill_max_requests=None, schedule_policy='fcfs', enable_priority_scheduling=False, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, schedule_conservativeness=1.0, page_size=1, hybrid_kvcache_ratio=None, swa_full_tokens_ratio=0.8, disable_hybrid_swa_memory=False, radix_eviction_policy='lru', device='cuda', tp_size=4, pp_size=1, pp_max_micro_batch_size=None, pp_async_batch_depth=0, stream_interval=1, stream_output=False, random_seed=395153307, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, watchdog_timeout=300, soft_watchdog_timeout=None, dist_timeout=None, download_dir=None, base_gpu_id=0, gpu_id_step=1, sleep_on_idle=False, custom_sigquit_handler=None, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, log_requests_format='text', crash_dump_folder=None, show_time_cost=False, enable_metrics=False, enable_metrics_for_all_schedulers=False, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, collect_tokens_histogram=False, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_trace=False, otlp_traces_endpoint='localhost:4317', export_metrics_to_file=False, export_metrics_to_file_dir=None, api_key=None, served_model_name='/p/scratch/envcomp/mihir/huggingface/Qwen3-30B-A3B', weight_version='default', chat_template=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser=None, tool_call_parser=None, tool_server=None, sampling_defaults='model', dp_size=1, load_balance_method='round_robin', prefill_round_robin_balance=False, dist_init_addr=None, nnodes=1, node_rank=0, json_model_override_args='{}', preferred_sampling_params=None, enable_lora=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, attention_backend='flashinfer', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='flashinfer', grammar_backend='xgrammar', mm_attention_backend=None, fp8_gemm_runner_backend='auto', nsa_prefill_backend='flashmla_sparse', nsa_decode_backend='fa3', disable_flashinfer_autotune=False, speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_token_map=None, speculative_attention_mode='prefill', speculative_draft_attention_backend=None, speculative_moe_runner_backend='auto', speculative_moe_a2a_backend=None, speculative_draft_model_quantization=None, speculative_ngram_min_match_window_size=1, speculative_ngram_max_match_window_size=12, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_branch_length=18, speculative_ngram_capacity=10000000, enable_multi_layer_eagle=False, ep_size=1, moe_a2a_backend='none', moe_runner_backend='auto', flashinfer_mxfp4_moe_precision='default', enable_flashinfer_allreduce_fusion=False, deepep_mode='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm=None, init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, enable_expert_distribution_metrics=False, deepep_config=None, moe_dense_tp_size=None, elastic_ep_backend=None, mooncake_ib_device=None, max_mamba_cache_size=None, mamba_ssm_dtype='float32', mamba_full_memory_ratio=0.9, mamba_scheduler_strategy='no_buffer', mamba_track_interval=256, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='layer_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='best_effort', hicache_storage_backend_extra_config=None, enable_lmcache=False, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, dllm_algorithm=None, dllm_algorithm_config=None, enable_double_sparsity=False, ds_channel_config_path=None, ds_heavy_channel_num=32, ds_heavy_token_num=256, ds_heavy_channel_type='qk', ds_sparse_decode_threshold=4096, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', multi_item_scoring_delimiter=None, disable_radix_cache=False, cuda_graph_max_bs=160, cuda_graph_bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160], disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, enable_layerwise_nvtx_marker=False, enable_nccl_nvls=False, enable_symm_mem=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, disable_outlines_disk_cache=False, disable_custom_all_reduce=False, enable_mscclpp=False, enable_torch_symm_mem=False, disable_overlap_schedule=False, enable_mixed_chunk=False, enable_dp_attention=False, enable_dp_lm_head=False, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, enable_torch_compile=False, enable_piecewise_cuda_graph=False, enable_torch_compile_debug_mode=False, torch_compile_max_bs=32, piecewise_cuda_graph_max_tokens=4096, piecewise_cuda_graph_tokens=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 640, 768, 896, 1024, 1152, 1280, 1408, 1536, 1664, 1792, 1920, 2048, 2176, 2304, 2432, 2560, 2688, 2816, 2944, 3072, 3200, 3328, 3456, 3584, 3712, 3840, 3968, 4096], piecewise_cuda_graph_compiler='eager', torchao_config='', enable_nan_detection=False, enable_p2p_check=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, num_continuous_decode_steps=1, delete_ckpt_after_loading=False, enable_memory_saver=False, enable_weights_cpu_backup=False, enable_draft_weights_cpu_backup=False, allow_auto_truncate=False, enable_custom_logit_processor=False, flashinfer_mla_disable_ragged=False, disable_shared_experts_fusion=False, disable_chunked_prefix_cache=False, disable_fast_image_processor=False, keep_mm_feature_on_device=False, enable_return_hidden_states=False, enable_return_routed_experts=False, scheduler_recv_interval=1, numa_node=None, enable_deterministic_inference=False, rl_on_policy_target=None, enable_attn_tp_input_scattered=False, enable_nsa_prefill_context_parallel=False, enable_fused_qk_norm_rope=False, enable_precise_embedding_interpolation=False, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, debug_tensor_dump_inject=False, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_decode_tp=None, disaggregation_decode_dp=None, disaggregation_prefill_pp=1, disaggregation_ib_device=None, disaggregation_decode_enable_offload_kvcache=False, disaggregation_decode_enable_fake_auto=False, num_reserved_decode_tokens=512, disaggregation_decode_polling_interval=1, encoder_only=False, language_only=False, encoder_transfer_backend='zmq_to_scheduler', encoder_urls=[], custom_weight_loader=[], weight_loader_disable_mmap=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, remote_instance_weight_loader_backend='nccl', remote_instance_weight_loader_start_seed_via_transfer_engine=False, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, mm_max_concurrent_calls=32, mm_per_request_timeout=10.0, enable_broadcast_mm_inputs_process=False, enable_prefix_mm_cache=False, mm_enable_dp_encoder=False, mm_process_config={}, limit_mm_data_per_request=None, decrypted_config_file=None, decrypted_draft_config_file=None, forward_hooks=None)
[2026-01-05 19:09:19] Using default HuggingFace chat template with detected content format: string
[2026-01-05 19:09:28] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:09:28] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:09:28] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:09:28] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:09:28] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:09:28] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:09:28] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:09:28] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:09:28] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:09:28] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:09:28] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:09:28] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:09:28] INFO utils.py:148: Note: detected 96 virtual cores but NumExpr set to maximum of 64, check "NUMEXPR_MAX_THREADS" environment variable.
[2026-01-05 19:09:28] INFO utils.py:151: Note: NumExpr detected 96 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
[2026-01-05 19:09:28] INFO utils.py:164: NumExpr defaulting to 16 threads.
[2026-01-05 19:09:31 TP1] Init torch distributed begin.
[2026-01-05 19:09:31 TP3] Init torch distributed begin.
[2026-01-05 19:09:31 TP2] Init torch distributed begin.
[2026-01-05 19:09:32 TP0] Init torch distributed begin.
[W105 19:09:32.865211493 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [localhost.localdomain]:30448 (errno: 97 - Address family not supported by protocol).
[W105 19:09:32.897607121 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [localhost.localdomain]:30448 (errno: 97 - Address family not supported by protocol).
[W105 19:09:32.905108701 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [localhost.localdomain]:30448 (errno: 97 - Address family not supported by protocol).
[W105 19:09:32.935072583 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [localhost.localdomain]:30448 (errno: 97 - Address family not supported by protocol).
[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[2026-01-05 19:09:33 TP0] sglang is using nccl==2.27.5
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3
[2026-01-05 19:09:33 TP0] Init torch distributed ends. mem usage=1.01 GB
[2026-01-05 19:09:33 TP2] Init torch distributed ends. mem usage=1.01 GB
[2026-01-05 19:09:33 TP1] Init torch distributed ends. mem usage=1.08 GB
[2026-01-05 19:09:33 TP3] Init torch distributed ends. mem usage=1.08 GB
[2026-01-05 19:09:36 TP2] Ignore import error when loading sglang.srt.models.glmasr: cannot import name 'GlmAsrConfig' from 'transformers' (/p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/transformers/__init__.py)
[2026-01-05 19:09:36 TP1] Ignore import error when loading sglang.srt.models.glmasr: cannot import name 'GlmAsrConfig' from 'transformers' (/p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/transformers/__init__.py)
[2026-01-05 19:09:36 TP0] Ignore import error when loading sglang.srt.models.glmasr: cannot import name 'GlmAsrConfig' from 'transformers' (/p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/transformers/__init__.py)
[2026-01-05 19:09:36 TP3] Ignore import error when loading sglang.srt.models.glmasr: cannot import name 'GlmAsrConfig' from 'transformers' (/p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/transformers/__init__.py)
[2026-01-05 19:09:37 TP1] Load weight begin. avail mem=37.99 GB
[2026-01-05 19:09:37 TP2] Load weight begin. avail mem=38.06 GB
[2026-01-05 19:09:37 TP0] Load weight begin. avail mem=38.06 GB
[2026-01-05 19:09:37 TP3] Load weight begin. avail mem=37.99 GB

Loading safetensors checkpoint shards:   0% Completed | 0/16 [00:00<?, ?it/s]

Loading safetensors checkpoint shards:   6% Completed | 1/16 [00:08<02:14,  8.97s/it]

Loading safetensors checkpoint shards:  12% Completed | 2/16 [00:17<02:05,  8.93s/it]

Loading safetensors checkpoint shards:  19% Completed | 3/16 [00:27<01:58,  9.11s/it]

Loading safetensors checkpoint shards:  25% Completed | 4/16 [00:36<01:52,  9.36s/it]

Loading safetensors checkpoint shards:  31% Completed | 5/16 [00:47<01:49,  9.97s/it]

Loading safetensors checkpoint shards:  38% Completed | 6/16 [00:58<01:42, 10.25s/it]

Loading safetensors checkpoint shards:  44% Completed | 7/16 [01:08<01:30, 10.09s/it]

Loading safetensors checkpoint shards:  50% Completed | 8/16 [01:17<01:17,  9.64s/it]

Loading safetensors checkpoint shards:  56% Completed | 9/16 [01:25<01:04,  9.22s/it]

Loading safetensors checkpoint shards:  62% Completed | 10/16 [01:33<00:53,  8.89s/it]

Loading safetensors checkpoint shards:  69% Completed | 11/16 [01:43<00:46,  9.25s/it]

Loading safetensors checkpoint shards:  75% Completed | 12/16 [01:52<00:36,  9.18s/it]

Loading safetensors checkpoint shards:  81% Completed | 13/16 [02:05<00:30, 10.31s/it]

Loading safetensors checkpoint shards:  88% Completed | 14/16 [02:17<00:21, 10.84s/it]

Loading safetensors checkpoint shards:  94% Completed | 15/16 [02:31<00:11, 11.86s/it]

Loading safetensors checkpoint shards: 100% Completed | 16/16 [02:33<00:00,  8.87s/it]

Loading safetensors checkpoint shards: 100% Completed | 16/16 [02:33<00:00,  9.62s/it]

[2026-01-05 19:12:11 TP3] Load weight end. type=Qwen3MoeForCausalLM, dtype=torch.bfloat16, avail mem=23.68 GB, mem usage=14.31 GB.
[2026-01-05 19:12:11 TP2] Load weight end. type=Qwen3MoeForCausalLM, dtype=torch.bfloat16, avail mem=23.75 GB, mem usage=14.31 GB.
[2026-01-05 19:12:11 TP1] Load weight end. type=Qwen3MoeForCausalLM, dtype=torch.bfloat16, avail mem=23.68 GB, mem usage=14.31 GB.
[2026-01-05 19:12:11 TP0] Load weight end. type=Qwen3MoeForCausalLM, dtype=torch.bfloat16, avail mem=23.75 GB, mem usage=14.31 GB.
[2026-01-05 19:12:11 TP0] Using KV cache dtype: torch.bfloat16
[2026-01-05 19:12:11 TP2] KV Cache is allocated. #tokens: 785681, K size: 8.99 GB, V size: 8.99 GB
[2026-01-05 19:12:11 TP2] Memory pool end. avail mem=5.12 GB
[2026-01-05 19:12:11 TP3] KV Cache is allocated. #tokens: 785681, K size: 8.99 GB, V size: 8.99 GB
[2026-01-05 19:12:11 TP3] Memory pool end. avail mem=5.05 GB
[2026-01-05 19:12:11 TP1] KV Cache is allocated. #tokens: 785681, K size: 8.99 GB, V size: 8.99 GB
[2026-01-05 19:12:11 TP1] Memory pool end. avail mem=5.05 GB
[2026-01-05 19:12:11 TP0] KV Cache is allocated. #tokens: 785681, K size: 8.99 GB, V size: 8.99 GB
[2026-01-05 19:12:11 TP0] Memory pool end. avail mem=5.12 GB
[2026-01-05 19:12:11 TP2] Capture cuda graph begin. This can take up to several minutes. avail mem=4.62 GB
[2026-01-05 19:12:11 TP3] Capture cuda graph begin. This can take up to several minutes. avail mem=4.54 GB
[2026-01-05 19:12:11 TP0] Capture cuda graph begin. This can take up to several minutes. avail mem=4.62 GB
[2026-01-05 19:12:11 TP0] Capture cuda graph bs [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160]
[2026-01-05 19:12:11 TP1] Capture cuda graph begin. This can take up to several minutes. avail mem=4.54 GB

  0%|          | 0/24 [00:00<?, ?it/s]
Capturing batches (bs=160 avail_mem=4.50 GB):   0%|          | 0/24 [00:00<?, ?it/s][2026-01-05 19:12:15 TP0] Using default MoE kernel config. Performance might be sub-optimal! Config file not found at /p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_5_1/E=128,N=192,device_name=NVIDIA_A100-SXM4-40GB.json, you can create them with https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton
[2026-01-05 19:12:15 TP0] Using MoE kernel config with down_moe=False. Performance might be sub-optimal! Config file not found at /p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_5_1/E=128,N=192,device_name=NVIDIA_A100-SXM4-40GB_down.json, you can create them with https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton
[2026-01-05 19:12:15 TP1] Using default MoE kernel config. Performance might be sub-optimal! Config file not found at /p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_5_1/E=128,N=192,device_name=NVIDIA_A100-SXM4-40GB.json, you can create them with https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton
[2026-01-05 19:12:15 TP1] Using MoE kernel config with down_moe=False. Performance might be sub-optimal! Config file not found at /p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_5_1/E=128,N=192,device_name=NVIDIA_A100-SXM4-40GB_down.json, you can create them with https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton
[2026-01-05 19:12:15 TP2] Using default MoE kernel config. Performance might be sub-optimal! Config file not found at /p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_5_1/E=128,N=192,device_name=NVIDIA_A100-SXM4-40GB.json, you can create them with https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton
[2026-01-05 19:12:15 TP2] Using MoE kernel config with down_moe=False. Performance might be sub-optimal! Config file not found at /p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_5_1/E=128,N=192,device_name=NVIDIA_A100-SXM4-40GB_down.json, you can create them with https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton
[2026-01-05 19:12:15 TP3] Using default MoE kernel config. Performance might be sub-optimal! Config file not found at /p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_5_1/E=128,N=192,device_name=NVIDIA_A100-SXM4-40GB.json, you can create them with https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton
[2026-01-05 19:12:15 TP3] Using MoE kernel config with down_moe=False. Performance might be sub-optimal! Config file not found at /p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/lib/python3.12/site-packages/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_5_1/E=128,N=192,device_name=NVIDIA_A100-SXM4-40GB_down.json, you can create them with https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton

Capturing batches (bs=160 avail_mem=4.50 GB):   4%|▍         | 1/24 [00:03<01:26,  3.76s/it]
Capturing batches (bs=152 avail_mem=4.34 GB):   4%|▍         | 1/24 [00:03<01:26,  3.76s/it]
Capturing batches (bs=152 avail_mem=4.34 GB):   8%|▊         | 2/24 [00:04<00:38,  1.76s/it]
Capturing batches (bs=144 avail_mem=4.31 GB):   8%|▊         | 2/24 [00:04<00:38,  1.76s/it]
Capturing batches (bs=144 avail_mem=4.31 GB):  12%|█▎        | 3/24 [00:04<00:21,  1.03s/it]
Capturing batches (bs=136 avail_mem=4.31 GB):  12%|█▎        | 3/24 [00:04<00:21,  1.03s/it]
Capturing batches (bs=136 avail_mem=4.31 GB):  17%|█▋        | 4/24 [00:04<00:13,  1.46it/s]
Capturing batches (bs=128 avail_mem=4.28 GB):  17%|█▋        | 4/24 [00:04<00:13,  1.46it/s]
Capturing batches (bs=128 avail_mem=4.28 GB):  21%|██        | 5/24 [00:04<00:09,  1.99it/s]
Capturing batches (bs=120 avail_mem=4.27 GB):  21%|██        | 5/24 [00:04<00:09,  1.99it/s]
Capturing batches (bs=120 avail_mem=4.27 GB):  25%|██▌       | 6/24 [00:04<00:06,  2.59it/s]
Capturing batches (bs=112 avail_mem=4.24 GB):  25%|██▌       | 6/24 [00:04<00:06,  2.59it/s]
Capturing batches (bs=112 avail_mem=4.24 GB):  29%|██▉       | 7/24 [00:04<00:05,  3.21it/s]
Capturing batches (bs=104 avail_mem=4.24 GB):  29%|██▉       | 7/24 [00:04<00:05,  3.21it/s]
Capturing batches (bs=104 avail_mem=4.24 GB):  33%|███▎      | 8/24 [00:05<00:04,  3.80it/s]
Capturing batches (bs=96 avail_mem=4.21 GB):  33%|███▎      | 8/24 [00:05<00:04,  3.80it/s] 
Capturing batches (bs=96 avail_mem=4.21 GB):  38%|███▊      | 9/24 [00:05<00:03,  4.31it/s]
Capturing batches (bs=88 avail_mem=4.20 GB):  38%|███▊      | 9/24 [00:05<00:03,  4.31it/s]
Capturing batches (bs=88 avail_mem=4.20 GB):  42%|████▏     | 10/24 [00:05<00:02,  4.80it/s]
Capturing batches (bs=80 avail_mem=4.17 GB):  42%|████▏     | 10/24 [00:05<00:02,  4.80it/s]
Capturing batches (bs=80 avail_mem=4.17 GB):  46%|████▌     | 11/24 [00:05<00:02,  5.14it/s]
Capturing batches (bs=72 avail_mem=4.16 GB):  46%|████▌     | 11/24 [00:05<00:02,  5.14it/s]
Capturing batches (bs=72 avail_mem=4.16 GB):  50%|█████     | 12/24 [00:05<00:02,  5.49it/s]
Capturing batches (bs=64 avail_mem=4.14 GB):  50%|█████     | 12/24 [00:05<00:02,  5.49it/s]
Capturing batches (bs=64 avail_mem=4.14 GB):  54%|█████▍    | 13/24 [00:05<00:01,  5.59it/s]
Capturing batches (bs=56 avail_mem=4.13 GB):  54%|█████▍    | 13/24 [00:05<00:01,  5.59it/s]
Capturing batches (bs=56 avail_mem=4.13 GB):  58%|█████▊    | 14/24 [00:06<00:01,  5.75it/s]
Capturing batches (bs=48 avail_mem=4.10 GB):  58%|█████▊    | 14/24 [00:06<00:01,  5.75it/s]
Capturing batches (bs=48 avail_mem=4.10 GB):  62%|██████▎   | 15/24 [00:06<00:01,  5.71it/s]
Capturing batches (bs=40 avail_mem=4.09 GB):  62%|██████▎   | 15/24 [00:06<00:01,  5.71it/s]
Capturing batches (bs=40 avail_mem=4.09 GB):  67%|██████▋   | 16/24 [00:06<00:01,  5.86it/s]
Capturing batches (bs=32 avail_mem=4.06 GB):  67%|██████▋   | 16/24 [00:06<00:01,  5.86it/s]
Capturing batches (bs=32 avail_mem=4.06 GB):  71%|███████   | 17/24 [00:07<00:02,  2.53it/s]
Capturing batches (bs=24 avail_mem=4.05 GB):  71%|███████   | 17/24 [00:07<00:02,  2.53it/s]
Capturing batches (bs=24 avail_mem=4.05 GB):  75%|███████▌  | 18/24 [00:08<00:03,  1.89it/s]
Capturing batches (bs=16 avail_mem=4.03 GB):  75%|███████▌  | 18/24 [00:08<00:03,  1.89it/s]
Capturing batches (bs=16 avail_mem=4.03 GB):  79%|███████▉  | 19/24 [00:09<00:03,  1.52it/s]
Capturing batches (bs=12 avail_mem=4.02 GB):  79%|███████▉  | 19/24 [00:09<00:03,  1.52it/s]
Capturing batches (bs=12 avail_mem=4.02 GB):  83%|████████▎ | 20/24 [00:09<00:02,  1.95it/s]
Capturing batches (bs=8 avail_mem=3.99 GB):  83%|████████▎ | 20/24 [00:09<00:02,  1.95it/s] 
Capturing batches (bs=8 avail_mem=3.99 GB):  88%|████████▊ | 21/24 [00:09<00:01,  2.41it/s]
Capturing batches (bs=4 avail_mem=3.98 GB):  88%|████████▊ | 21/24 [00:09<00:01,  2.41it/s]
Capturing batches (bs=4 avail_mem=3.98 GB):  92%|█████████▏| 22/24 [00:09<00:00,  2.92it/s]
Capturing batches (bs=2 avail_mem=3.96 GB):  92%|█████████▏| 22/24 [00:09<00:00,  2.92it/s]
Capturing batches (bs=2 avail_mem=3.96 GB):  96%|█████████▌| 23/24 [00:09<00:00,  3.41it/s]
Capturing batches (bs=1 avail_mem=3.95 GB):  96%|█████████▌| 23/24 [00:09<00:00,  3.41it/s]
Capturing batches (bs=1 avail_mem=3.95 GB): 100%|██████████| 24/24 [00:11<00:00,  1.78it/s]
Capturing batches (bs=1 avail_mem=3.95 GB): 100%|██████████| 24/24 [00:11<00:00,  2.18it/s]
[2026-01-05 19:12:23 TP0] Registering 2328 cuda graph addresses
[2026-01-05 19:12:23 TP0] Capture cuda graph end. Time elapsed: 12.00 s. mem usage=0.70 GB. avail mem=3.92 GB.
[2026-01-05 19:12:23 TP2] Capture cuda graph end. Time elapsed: 12.09 s. mem usage=0.70 GB. avail mem=3.92 GB.
[2026-01-05 19:12:23 TP1] Capture cuda graph end. Time elapsed: 12.09 s. mem usage=0.70 GB. avail mem=3.85 GB.
[2026-01-05 19:12:23 TP3] Capture cuda graph end. Time elapsed: 12.13 s. mem usage=0.70 GB. avail mem=3.85 GB.
[2026-01-05 19:12:24 TP0] max_total_num_tokens=785681, chunked_prefill_size=4096, max_prefill_tokens=16384, max_running_requests=4096, context_len=40960, available_gpu_mem=3.92 GB
[2026-01-05 19:12:24] INFO:     Started server process [3020722]
[2026-01-05 19:12:24] INFO:     Waiting for application startup.
[2026-01-05 19:12:24] Using default chat sampling params from model generation config: {'repetition_penalty': 1.0, 'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
[2026-01-05 19:12:24] Using default chat sampling params from model generation config: {'repetition_penalty': 1.0, 'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
[2026-01-05 19:12:24] INFO:     Application startup complete.
[2026-01-05 19:12:24] INFO:     Uvicorn running on http://localhost:30000 (Press CTRL+C to quit)
[2026-01-05 19:12:25] INFO:     127.0.0.1:43960 - "GET /model_info HTTP/1.1" 200 OK
[2026-01-05 19:12:25 TP0] Prefill batch, #new-seq: 1, #new-token: 6, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, 
[2026-01-05 19:12:26] INFO:     127.0.0.1:43972 - "POST /generate HTTP/1.1" 200 OK
[2026-01-05 19:12:26] The server is fired up and ready to roll!
[2026-01-05 19:12:31] INFO:     127.0.0.1:43974 - "GET /v1/models HTTP/1.1" 200 OK
Using server at http://localhost:30000/v1
Launching sglang server:
  /p/project1/envcomp/mahajan1/miles/tab-model-eval/.venv/bin/python -m sglang.launch_server --model-path /p/scratch/envcomp/mihir/huggingface/Qwen3-30B-A3B --host localhost --port 30000 --context-length 40960 --mem-fraction-static 0.85 --tp-size 4
Waiting for server at http://localhost:30000 ...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server not ready yet (All connection attempts failed); retrying...
Server is up.

############################################################
# BATCH EVALUATION MODE
# Processing 11 evaluation job(s)
############################################################

  [1/11] Step 0: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_0_generati
  0%|          | 0/5 [00:00<?, ?it/s]ons.jsonl
  [2/11] Step 500: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_500_generations.jsonl
  [3/11] Step 1000: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_1000_generations.jsonl
  [4/11] Step 1500: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_1500_generations.jsonl
  [5/11] Step 2000: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_2000_generations.jsonl
  [6/11] Step 2500: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_2500_generations.jsonl
  [7/11] Step 3000: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_3000_generations.jsonl
  [8/11] Step 3500: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_3500_generations.jsonl
  [9/11] Step 4000: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_4000_generations.jsonl
  [10/11] Step 4500: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_4500_generations.jsonl
  [11/11] Step 5000: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_5000_generations.jsonl

LocalLogger initialized. Logs will be saved to: data/eval/local_logs/eval_13032816

[1/11] Processing step 0...

============================================================
Evaluating: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_0_generations.jsonl
Output: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_0_evaluations.jsonl
Step: 0
============================================================

Filtered dataset:
  Valid test cases: 5
  Skipped (too long): 0

Running 5 test cases with concurrency=4 ...
[2026-01-05 19:12:32 TP0] Prefill batch, #new-seq: 1, #new-token: 4096, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 3, 
[2026-01-05 19:12:33 TP0] Prefill batch, #new-seq: 2, #new-token: 4096, #cached-token: 218, token usage: 0.01, #running-req: 0, #queue-req: 2, 
[2026-01-05 19:12:33 TP0] Prefill batch, #new-seq: 1, #new-token: 4096, #cached-token: 0, token usage: 0.01, #running-req: 1, #queue-req: 2, 
[2026-01-05 19:12:33 TP0] Prefill batch, #new-seq: 3, #new-token: 4083, #cached-token: 6300, token usage: 0.02, #running-req: 1, #queue-req: 0, 
[2026-01-05 19:12:34 TP0] Decode batch, #running-req: 4, #token: 16503, token usage: 0.02, cuda graph: True, gen throughput (token/s): 0.74, #queue-req: 0, 
[2026-01-05 19:12:34] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:34 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 7090, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:35] INFO:     127.0.0.1:44014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:35 TP0] Prefill batch, #new-seq: 1, #new-token: 66, #cached-token: 6129, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:35 TP0] Decode batch, #running-req: 4, #token: 16519, token usage: 0.02, cuda graph: True, gen throughput (token/s): 264.88, #queue-req: 0, 
[2026-01-05 19:12:35] INFO:     127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:35 TP0] Prefill batch, #new-seq: 1, #new-token: 113, #cached-token: 8079, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:35] INFO:     127.0.0.1:44006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:35 TP0] Prefill batch, #new-seq: 1, #new-token: 95, #cached-token: 1302, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:35] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:35 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 7090, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:36 TP0] Decode batch, #running-req: 4, #token: 16497, token usage: 0.02, cuda graph: True, gen throughput (token/s): 181.26, #queue-req: 0, 
[2026-01-05 19:12:36] INFO:     127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:36 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 8191, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:36] INFO:     127.0.0.1:44014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:36 TP0] Prefill batch, #new-seq: 1, #new-token: 84, #cached-token: 6134, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:36] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:36 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 7090, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:36] INFO:     127.0.0.1:44006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:36 TP0] Decode batch, #running-req: 4, #token: 15288, token usage: 0.02, cuda graph: True, gen throughput (token/s): 310.60, #queue-req: 0, 
[2026-01-05 19:12:36 TP0] Prefill batch, #new-seq: 1, #new-token: 80, #cached-token: 1289, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:36] INFO:     127.0.0.1:44014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:36 TP0] Prefill batch, #new-seq: 1, #new-token: 98, #cached-token: 6134, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:36] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:37] INFO:     127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:37 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 7090, token usage: 0.01, #running-req: 2, #queue-req: 0, 
[2026-01-05 19:12:37 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 8191, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:37 TP0] Decode batch, #running-req: 4, #token: 16450, token usage: 0.02, cuda graph: True, gen throughput (token/s): 280.04, #queue-req: 0, 
[2026-01-05 19:12:37] INFO:     127.0.0.1:44014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:37 TP0] Prefill batch, #new-seq: 1, #new-token: 85, #cached-token: 6138, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:37] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:37 TP0] Prefill batch, #new-seq: 1, #new-token: 100, #cached-token: 6933, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:37] INFO:     127.0.0.1:44006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:37 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1368, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:37 TP0] Decode batch, #running-req: 4, #token: 16391, token usage: 0.02, cuda graph: True, gen throughput (token/s): 324.07, #queue-req: 0, 
[2026-01-05 19:12:37] INFO:     127.0.0.1:44014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:37 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 6231, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:37] INFO:     127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:37 TP0] Prefill batch, #new-seq: 1, #new-token: 114, #cached-token: 8078, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:38 TP0] Decode batch, #running-req: 4, #token: 16433, token usage: 0.02, cuda graph: True, gen throughput (token/s): 373.62, #queue-req: 0, 
[2026-01-05 19:12:38] INFO:     127.0.0.1:44014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:38 TP0] Prefill batch, #new-seq: 1, #new-token: 99, #cached-token: 6135, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:38] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:38 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 7090, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:38] INFO:     127.0.0.1:44006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:38 TP0] Prefill batch, #new-seq: 1, #new-token: 80, #cached-token: 1289, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:38 TP0] Decode batch, #running-req: 4, #token: 16484, token usage: 0.02, cuda graph: True, gen throughput (token/s): 322.91, #queue-req: 0, 
[2026-01-05 19:12:38] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:38 TP0] Prefill batch, #new-seq: 1, #new-token: 109, #cached-token: 6933, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:38] INFO:     127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:38 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 8191, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:38] INFO:     127.0.0.1:44006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:38 TP0] Decode batch, #running-req: 4, #token: 15275, token usage: 0.02, cuda graph: True, gen throughput (token/s): 371.98, #queue-req: 0, 
[2026-01-05 19:12:38 TP0] Prefill batch, #new-seq: 1, #new-token: 78, #cached-token: 1291, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:39] INFO:     127.0.0.1:44014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:39 TP0] Prefill batch, #new-seq: 1, #new-token: 84, #cached-token: 6137, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:39 TP0] Decode batch, #running-req: 4, #token: 16473, token usage: 0.02, cuda graph: True, gen throughput (token/s): 381.46, #queue-req: 0, 
[2026-01-05 19:12:39] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:39 TP0] Prefill batch, #new-seq: 1, #new-token: 66, #cached-token: 6928, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:39] INFO:     127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:39 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 8191, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:39] INFO:     127.0.0.1:44006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:39 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1396, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:39] INFO:     127.0.0.1:44014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:39 TP0] Prefill batch, #new-seq: 1, #new-token: 78, #cached-token: 6142, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:39 TP0] Decode batch, #running-req: 4, #token: 16362, token usage: 0.02, cuda graph: True, gen throughput (token/s): 280.14, #queue-req: 0, 
[2026-01-05 19:12:39] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:39 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 7090, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:40] INFO:     127.0.0.1:44014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:40 TP0] Prefill batch, #new-seq: 1, #new-token: 68, #cached-token: 6134, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:40 TP0] Decode batch, #running-req: 3, #token: 16506, token usage: 0.02, cuda graph: True, gen throughput (token/s): 377.04, #queue-req: 0, 
[2026-01-05 19:12:40] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK

 20%|██        | 1/5 [00:08<00:33,  8.40s/it][2026-01-05 19:12:40 TP0] Prefill batch, #new-seq: 1, #new-token: 344, #cached-token: 1223, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:40] INFO:     127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:40 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 8191, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:40] INFO:     127.0.0.1:44006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:40 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1368, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:40 TP0] Decode batch, #running-req: 4, #token: 15783, token usage: 0.02, cuda graph: True, gen throughput (token/s): 322.79, #queue-req: 0, 
[2026-01-05 19:12:40] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:40 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1566, token usage: 0.02, #running-req: 3, #queue-req: 0, 
[2026-01-05 19:12:40] INFO:     127.0.0.1:44014 - "POST /v1/chat/completions HTTP/1.1" 200 OK

 40%|████      | 2/5 [00:08<00:11,  3.80s/it][2026-01-05 19:12:41 TP0] Decode batch, #running-req: 3, #token: 9841, token usage: 0.01, cuda graph: True, gen throughput (token/s): 390.17, #queue-req: 0, 
[2026-01-05 19:12:41] INFO:     127.0.0.1:44006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:41 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1368, token usage: 0.01, #running-req: 2, #queue-req: 0, 
[2026-01-05 19:12:41] INFO:     127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:41] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:41 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1566, token usage: 0.00, #running-req: 1, #queue-req: 0, 
[2026-01-05 19:12:41 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 8191, token usage: 0.01, #running-req: 2, #queue-req: 0, 
[2026-01-05 19:12:41 TP0] Decode batch, #running-req: 3, #token: 9783, token usage: 0.01, cuda graph: True, gen throughput (token/s): 245.33, #queue-req: 0, 
[2026-01-05 19:12:41] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:41 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1566, token usage: 0.01, #running-req: 2, #queue-req: 0, 
[2026-01-05 19:12:41 TP0] Decode batch, #running-req: 3, #token: 9864, token usage: 0.01, cuda graph: True, gen throughput (token/s): 339.09, #queue-req: 0, 
[2026-01-05 19:12:41] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:42 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1566, token usage: 0.01, #running-req: 2, #queue-req: 0, 
[2026-01-05 19:12:42] INFO:     127.0.0.1:44006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:42 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1368, token usage: 0.01, #running-req: 2, #queue-req: 0, 
[2026-01-05 19:12:42] INFO:     127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:42 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 8191, token usage: 0.01, #running-req: 2, #queue-req: 0, 
[2026-01-05 19:12:42 TP0] Decode batch, #running-req: 3, #token: 9774, token usage: 0.01, cuda graph: True, gen throughput (token/s): 243.92, #queue-req: 0, 
[2026-01-05 19:12:42] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:42 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1566, token usage: 0.01, #running-req: 2, #queue-req: 0, 
[2026-01-05 19:12:42 TP0] Decode batch, #running-req: 3, #token: 9855, token usage: 0.01, cuda graph: True, gen throughput (token/s): 338.42, #queue-req: 0, 
[2026-01-05 19:12:42] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:42 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1566, token usage: 0.01, #running-req: 2, #queue-req: 0, 
[2026-01-05 19:12:42] INFO:     127.0.0.1:43992 - "POST /v1/chat/completions HTTP/1.1" 200 OK

 60%|██████    | 3/5 [00:10<00:05,  2.94s/it][2026-01-05 19:12:42] INFO:     127.0.0.1:44006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:43 TP0] Decode batch, #running-req: 1, #token: 0, token usage: 0.00, cuda graph: True, gen throughput (token/s): 224.90, #queue-req: 0, 
[2026-01-05 19:12:43] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:43 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1566, token usage: 0.00, #running-req: 0, #queue-req: 0, 
[2026-01-05 19:12:43] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:43 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1566, token usage: 0.00, #running-req: 0, #queue-req: 0, 
[2026-01-05 19:12:43 TP0] Decode batch, #running-req: 1, #token: 1570, token usage: 0.00, cuda graph: True, gen throughput (token/s): 124.63, #queue-req: 0, 
[2026-01-05 19:12:43] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
[2026-01-05 19:12:43 TP0] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 1566, token usage: 0.00, #running-req: 0, #queue-req: 0, 
[2026-01-05 19:12:43 TP0] Decode batch, #running-req: 1, #token: 1573, token usage: 0.00, cuda graph: True, gen throughput (token/s): 153.80, #queue-req: 0, 
[2026-01-05 19:12:43] INFO:     127.0.0.1:43976 - "POST /v1/chat/completions HTTP/1.1" 200 OK

100%|██████████| 5/5 [00:11<00:00,  1.52s/it]
100%|██████████| 5/5 [00:11<00:00,  2.35s/it]
[2026-01-05 19:12:43] SIGTERM received. signum=None frame=None. Draining requests and shutting down...
[2026-01-05 19:12:45] Gracefully exiting... Remaining number of requests 0. Remaining requests remaining_rids=[].
Logged metrics to data/eval/local_logs/eval_13032816/metrics.jsonl: eval_step=0

==================================================
--- Evaluation Complete (step 0) ---
==================================================
Total Test Cases: 5
Total Judge Pass At N: 3
Total Judge Avg At N: 40.00%
Total Exact Match Pass At N: 3
Total Exact Match Avg At N: 38.00%
Evaluations output file: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_0_evaluations.jsonl

[2/11] Processing step 500...

============================================================
Evaluating: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_500_generations.jsonl
Output: data/eval/output/13032816/qwen8B_lora_bs_32_13032816_500_evaluations.jsonl
Step: 500
============================================================
Shutting down sglang server ...
Traceback (most recent call last):
  File "/p/project1/envcomp/mahajan1/miles/tab-model-eval/src/evaluation/sglang_eval.py", line 696, in <module>
    asyncio.run(amain(args))
  File "/p/home/jusers/mahajan1/juwels/.local/share/uv/python/cpython-3.12.10-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 195, in run
    return runner.run(main)
           ^^^^^^^^^^^^^^^^
  File "/p/home/jusers/mahajan1/juwels/.local/share/uv/python/cpython-3.12.10-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 118, in run
    return self._loop.run_until_complete(task)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/home/jusers/mahajan1/juwels/.local/share/uv/python/cpython-3.12.10-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
    return future.result()
           ^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/tab-model-eval/src/evaluation/sglang_eval.py", line 681, in amain
    await run_batch_eval(args, base_url=base_url)
  File "/p/project1/envcomp/mahajan1/miles/tab-model-eval/src/evaluation/sglang_eval.py", line 565, in run_batch_eval
    result = await run_single_eval(
             ^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/tab-model-eval/src/evaluation/sglang_eval.py", line 363, in run_single_eval
    loaded_data = load_dataset(generations_file)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/project1/envcomp/mahajan1/miles/tab-model-eval/src/evaluation/sglang_eval.py", line 154, in load_dataset
    with open(filepath, "r") as f:
         ^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'data/eval/output/13032816/qwen8B_lora_bs_32_13032816_500_generations.jsonl'

==============================================
Evaluation complete. Local logs saved to:
  data/eval/local_logs/eval_13032816/

To sync to wandb from the login node, run:
  python src/evaluation/sync_local_logs_to_wandb.py --log_dir data/eval/local_logs/eval_13032816
==============================================