[terminal_command] TERMINAL ()
[2025-11-30 12:16:39] INFO:     10.86.2.251:42424 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:39] Decode batch, #running-req: 12, #token: 11040, token usage: 0.02, cuda graph: True, gen throughput (token/s): 4531.45, #queue-req: 0, [2025-11-30 12:16:39] INFO:     10.86.2.251:42414 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:39] Decode batch, #running-req: 11, #token: 11215, token usage: 0.02, cuda graph: True, gen throughput (token/s): 4135.84, #queue-req: 0, [2025-11-30 12:16:39] Decode batch, #running-req: 11, #token: 11655, token usage: 0.02, cuda graph: True, gen throughput (token/s): 3977.29, #queue-req: 0, [2025-11-30 12:16:39] INFO:     10.86.2.251:42136 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:39] Decode batch, #running-req: 10, #token: 9701, token usage: 0.02, cuda graph: True, gen throughput (token/s): 3823.22, #queue-req: 0, [2025-11-30 12:16:39] INFO:     10.86.2.251:42272 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:39] Prefill batch, #new-seq: 1, #new-token: 51, #cached-token: 276, token usage: 0.01, #running-req: 9, #queue-req: 0, [2025-11-30 12:16:39] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 326, token usage: 0.01, #running-req: 10, #queue-req: 0, [2025-11-30 12:16:40] Decode batch, #running-req: 11, #token: 8581, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3061.69, #queue-req: 0, [2025-11-30 12:16:40] INFO:     10.86.2.251:42372 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:40] Decode batch, #running-req: 10, #token: 8113, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4179.76, #queue-req: 0, [2025-11-30 12:16:40] Decode batch, #running-req: 10, #token: 8513, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3928.87, #queue-req: 0, [2025-11-30 12:16:40] INFO:     10.86.2.251:42410 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:40] Decode batch, #running-req: 9, #token: 8238, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3817.58, #queue-req: 0, [2025-11-30 12:16:40] INFO:     10.86.2.251:42328 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:40] INFO:     10.86.2.251:42440 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:40] INFO:     10.86.2.251:42384 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:40] Prefill batch, #new-seq: 2, #new-token: 54, #cached-token: 602, token usage: 0.01, #running-req: 6, #queue-req: 0, [2025-11-30 12:16:40] Decode batch, #running-req: 8, #token: 5969, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2998.47, #queue-req: 0, [2025-11-30 12:16:40] Prefill batch, #new-seq: 2, #new-token: 54, #cached-token: 604, token usage: 0.01, #running-req: 8, #queue-req: 0, [2025-11-30 12:16:40] Decode batch, #running-req: 10, #token: 6374, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3398.44, #queue-req: 0, [2025-11-30 12:16:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 328, token usage: 0.01, #running-req: 10, #queue-req: 0, [2025-11-30 12:16:40] Prefill batch, #new-seq: 1, #new-token: 53, #cached-token: 276, token usage: 0.01, #running-req: 11, #queue-req: 0, [2025-11-30 12:16:40] Decode batch, #running-req: 12, #token: 6855, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3532.06, #queue-req: 0, [2025-11-30 12:16:40] Decode batch, #running-req: 12, #token: 7335, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4722.99, #queue-req: 0, [2025-11-30 12:16:40] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 328, token usage: 0.01, #running-req: 12, #queue-req: 0, [2025-11-30 12:16:40] Prefill batch, #new-seq: 3, #new-token: 159, #cached-token: 828, token usage: 0.01, #running-req: 13, #queue-req: 0, [2025-11-30 12:16:40] Decode batch, #running-req: 16, #token: 8021, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4514.85, #queue-req: 0, [2025-11-30 12:16:40] INFO:     10.86.2.251:42334 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:40] Decode batch, #running-req: 15, #token: 7090, token usage: 0.01, cuda graph: True, gen throughput (token/s): 6040.79, #queue-req: 0, [2025-11-30 12:16:41] Decode batch, #running-req: 15, #token: 7690, token usage: 0.01, cuda graph: True, gen throughput (token/s): 5649.82, #queue-req: 0, [2025-11-30 12:16:41] Prefill batch, #new-seq: 2, #new-token: 55, #cached-token: 604, token usage: 0.01, #running-req: 15, #queue-req: 0, [2025-11-30 12:16:41] Prefill batch, #new-seq: 2, #new-token: 108, #cached-token: 552, token usage: 0.01, #running-req: 17, #queue-req: 0, [2025-11-30 12:16:41] INFO:     10.86.2.251:42368 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:41] Decode batch, #running-req: 18, #token: 7074, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4640.45, #queue-req: 0, [2025-11-30 12:16:41] Decode batch, #running-req: 18, #token: 7794, token usage: 0.01, cuda graph: True, gen throughput (token/s): 6943.02, #queue-req: 0, [2025-11-30 12:16:41] Decode batch, #running-req: 18, #token: 8514, token usage: 0.01, cuda graph: True, gen throughput (token/s): 6686.26, #queue-req: 0, [2025-11-30 12:16:41] INFO:     10.86.2.251:42388 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:41] Decode batch, #running-req: 17, #token: 7954, token usage: 0.01, cuda graph: True, gen throughput (token/s): 6526.56, #queue-req: 0, [2025-11-30 12:16:41] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 329, token usage: 0.01, #running-req: 17, #queue-req: 0, [2025-11-30 12:16:41] Decode batch, #running-req: 18, #token: 8657, token usage: 0.01, cuda graph: True, gen throughput (token/s): 6087.29, #queue-req: 0, [2025-11-30 12:16:41] INFO:     10.86.2.251:42464 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:41] Decode batch, #running-req: 17, #token: 8895, token usage: 0.01, cuda graph: True, gen throughput (token/s): 6482.67, #queue-req: 0, [2025-11-30 12:16:41] Decode batch, #running-req: 17, #token: 9575, token usage: 0.02, cuda graph: True, gen throughput (token/s): 6161.90, #queue-req: 0, [2025-11-30 12:16:41] INFO:     10.86.2.251:42500 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:41] Decode batch, #running-req: 16, #token: 9761, token usage: 0.02, cuda graph: True, gen throughput (token/s): 5937.18, #queue-req: 0, [2025-11-30 12:16:42] Decode batch, #running-req: 16, #token: 10401, token usage: 0.02, cuda graph: True, gen throughput (token/s): 5746.70, #queue-req: 0, [2025-11-30 12:16:42] Decode batch, #running-req: 16, #token: 11041, token usage: 0.02, cuda graph: True, gen throughput (token/s): 5635.66, #queue-req: 0, [2025-11-30 12:16:42] INFO:     10.86.2.251:42444 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:42] Decode batch, #running-req: 15, #token: 10834, token usage: 0.02, cuda graph: True, gen throughput (token/s): 5343.52, #queue-req: 0, [2025-11-30 12:16:42] Decode batch, #running-req: 15, #token: 11434, token usage: 0.02, cuda graph: True, gen throughput (token/s): 5247.29, #queue-req: 0, [2025-11-30 12:16:42] Decode batch, #running-req: 15, #token: 12034, token usage: 0.02, cuda graph: True, gen throughput (token/s): 5152.06, #queue-req: 0, [2025-11-30 12:16:42] INFO:     10.86.2.251:42578 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:42] Decode batch, #running-req: 14, #token: 11389, token usage: 0.02, cuda graph: True, gen throughput (token/s): 5107.61, #queue-req: 0, [2025-11-30 12:16:42] INFO:     10.86.2.251:42528 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:42] INFO:     10.86.2.251:42456 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:42] INFO:     10.86.2.251:42476 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:42] INFO:     10.86.2.251:42396 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:42] Decode batch, #running-req: 10, #token: 8590, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4243.26, #queue-req: 0, [2025-11-30 12:16:42] INFO:     10.86.2.251:42338 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:42] Decode batch, #running-req: 9, #token: 6734, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3686.47, #queue-req: 0, [2025-11-30 12:16:42] Decode batch, #running-req: 9, #token: 7094, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3527.50, #queue-req: 0, [2025-11-30 12:16:43] Decode batch, #running-req: 9, #token: 7454, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3523.07, #queue-req: 0, [2025-11-30 12:16:43] Decode batch, #running-req: 9, #token: 7814, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3525.52, #queue-req: 0, [2025-11-30 12:16:43] Decode batch, #running-req: 9, #token: 8174, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3505.41, #queue-req: 0, [2025-11-30 12:16:43] Decode batch, #running-req: 9, #token: 8534, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3492.63, #queue-req: 0, [2025-11-30 12:16:43] Decode batch, #running-req: 9, #token: 8894, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3431.27, #queue-req: 0, [2025-11-30 12:16:43] INFO:     10.86.2.251:42522 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:43] Decode batch, #running-req: 8, #token: 8229, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3362.58, #queue-req: 0, [2025-11-30 12:16:43] Decode batch, #running-req: 8, #token: 8549, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3346.89, #queue-req: 0, [2025-11-30 12:16:43] INFO:     10.86.2.251:42514 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:43] Decode batch, #running-req: 7, #token: 7642, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3235.99, #queue-req: 0, [2025-11-30 12:16:43] Decode batch, #running-req: 7, #token: 7922, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2937.88, #queue-req: 0, [2025-11-30 12:16:43] INFO:     10.86.2.251:42572 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:43] INFO:     10.86.2.251:42486 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:43] Decode batch, #running-req: 5, #token: 5805, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2581.74, #queue-req: 0, [2025-11-30 12:16:44] Decode batch, #running-req: 5, #token: 6005, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2198.27, #queue-req: 0, [2025-11-30 12:16:44] Decode batch, #running-req: 5, #token: 6205, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2177.17, #queue-req: 0, [2025-11-30 12:16:44] Decode batch, #running-req: 5, #token: 6405, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2169.47, #queue-req: 0, [2025-11-30 12:16:44] Decode batch, #running-req: 5, #token: 6605, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2161.66, #queue-req: 0, [2025-11-30 12:16:44] Decode batch, #running-req: 5, #token: 6805, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2161.22, #queue-req: 0, [2025-11-30 12:16:44] Decode batch, #running-req: 5, #token: 7005, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2153.69, #queue-req: 0, [2025-11-30 12:16:44] Prefill batch, #new-seq: 2, #new-token: 21, #cached-token: 640, token usage: 0.01, #running-req: 5, #queue-req: 0, [2025-11-30 12:16:44] Decode batch, #running-req: 7, #token: 7313, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2416.49, #queue-req: 0, [2025-11-30 12:16:44] Decode batch, #running-req: 7, #token: 7593, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2895.61, #queue-req: 0, [2025-11-30 12:16:44] Decode batch, #running-req: 7, #token: 7873, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2830.84, #queue-req: 0, [2025-11-30 12:16:44] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 329, token usage: 0.01, #running-req: 7, #queue-req: 0, [2025-11-30 12:16:44] Decode batch, #running-req: 8, #token: 8182, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2882.09, #queue-req: 0, [2025-11-30 12:16:44] INFO:     10.86.2.251:42550 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:45] Decode batch, #running-req: 7, #token: 5535, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3096.74, #queue-req: 0, [2025-11-30 12:16:45] INFO:     10.86.2.251:56610 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:45] INFO:     10.86.2.251:42538 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:45] Decode batch, #running-req: 5, #token: 4056, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2456.49, #queue-req: 0, [2025-11-30 12:16:45] Decode batch, #running-req: 5, #token: 4256, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2257.03, #queue-req: 0, [2025-11-30 12:16:45] Decode batch, #running-req: 5, #token: 4456, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2260.54, #queue-req: 0, [2025-11-30 12:16:45] Decode batch, #running-req: 5, #token: 4656, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2255.06, #queue-req: 0, [2025-11-30 12:16:45] Decode batch, #running-req: 5, #token: 4856, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2240.40, #queue-req: 0, [2025-11-30 12:16:45] INFO:     10.86.2.251:56620 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:45] Decode batch, #running-req: 4, #token: 4629, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1989.34, #queue-req: 0, [2025-11-30 12:16:45] Decode batch, #running-req: 4, #token: 4789, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1820.48, #queue-req: 0, [2025-11-30 12:16:45] Decode batch, #running-req: 4, #token: 4949, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1809.73, #queue-req: 0, [2025-11-30 12:16:45] Decode batch, #running-req: 4, #token: 5109, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1810.18, #queue-req: 0, [2025-11-30 12:16:45] Decode batch, #running-req: 4, #token: 5269, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1809.35, #queue-req: 0, [2025-11-30 12:16:46] Decode batch, #running-req: 4, #token: 5429, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1791.29, #queue-req: 0, [2025-11-30 12:16:46] INFO:     10.86.2.251:42556 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:46] Decode batch, #running-req: 3, #token: 3568, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1453.79, #queue-req: 0, [2025-11-30 12:16:46] Decode batch, #running-req: 3, #token: 3688, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1399.92, #queue-req: 0, [2025-11-30 12:16:46] Decode batch, #running-req: 3, #token: 3808, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1393.15, #queue-req: 0, [2025-11-30 12:16:46] INFO:     10.86.2.251:42564 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:46] Decode batch, #running-req: 2, #token: 1840, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1158.33, #queue-req: 0, [2025-11-30 12:16:46] Decode batch, #running-req: 2, #token: 1920, token usage: 0.00, cuda graph: True, gen throughput (token/s): 981.30, #queue-req: 0, [2025-11-30 12:16:46] Decode batch, #running-req: 2, #token: 2000, token usage: 0.00, cuda graph: True, gen throughput (token/s): 980.29, #queue-req: 0, [2025-11-30 12:16:46] Decode batch, #running-req: 2, #token: 2080, token usage: 0.00, cuda graph: True, gen throughput (token/s): 974.80, #queue-req: 0, [2025-11-30 12:16:46] Decode batch, #running-req: 2, #token: 2160, token usage: 0.00, cuda graph: True, gen throughput (token/s): 973.65, #queue-req: 0, [2025-11-30 12:16:46] Decode batch, #running-req: 2, #token: 2240, token usage: 0.00, cuda graph: True, gen throughput (token/s): 972.23, #queue-req: 0, [2025-11-30 12:16:46] Decode batch, #running-req: 2, #token: 2320, token usage: 0.00, cuda graph: True, gen throughput (token/s): 969.81, #queue-req: 0, [2025-11-30 12:16:46] Decode batch, #running-req: 2, #token: 2400, token usage: 0.00, cuda graph: True, gen throughput (token/s): 966.93, #queue-req: 0, [2025-11-30 12:16:47] Decode batch, #running-req: 2, #token: 2480, token usage: 0.00, cuda graph: True, gen throughput (token/s): 967.36, #queue-req: 0, [2025-11-30 12:16:47] Decode batch, #running-req: 2, #token: 2560, token usage: 0.00, cuda graph: True, gen throughput (token/s): 965.48, #queue-req: 0, [2025-11-30 12:16:47] Decode batch, #running-req: 2, #token: 2640, token usage: 0.00, cuda graph: True, gen throughput (token/s): 962.53, #queue-req: 0, [2025-11-30 12:16:47] Decode batch, #running-req: 2, #token: 2720, token usage: 0.00, cuda graph: True, gen throughput (token/s): 959.79, #queue-req: 0, [2025-11-30 12:16:47] Decode batch, #running-req: 2, #token: 2800, token usage: 0.00, cuda graph: True, gen throughput (token/s): 957.52, #queue-req: 0, [2025-11-30 12:16:47] Decode batch, #running-req: 2, #token: 2880, token usage: 0.00, cuda graph: True, gen throughput (token/s): 955.64, #queue-req: 0, [2025-11-30 12:16:47] INFO:     10.86.2.251:56636 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:16:47] INFO:     10.86.2.251:56622 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:25:11] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 326, token usage: 0.00, #running-req: 0, #queue-req: 0, [2025-11-30 12:25:11] Decode batch, #running-req: 1, #token: 331, token usage: 0.00, cuda graph: True, gen throughput (token/s): 0.14, #queue-req: 0, [2025-11-30 12:25:12] Decode batch, #running-req: 1, #token: 371, token usage: 0.00, cuda graph: True, gen throughput (token/s): 530.34, #queue-req: 0, [2025-11-30 12:25:12] Decode batch, #running-req: 1, #token: 411, token usage: 0.00, cuda graph: True, gen throughput (token/s): 534.49, #queue-req: 0, [2025-11-30 12:25:12] Decode batch, #running-req: 1, #token: 451, token usage: 0.00, cuda graph: True, gen throughput (token/s): 537.43, #queue-req: 0, [2025-11-30 12:25:12] Decode batch, #running-req: 1, #token: 491, token usage: 0.00, cuda graph: True, gen throughput (token/s): 538.57, #queue-req: 0, [2025-11-30 12:25:12] Decode batch, #running-req: 1, #token: 531, token usage: 0.00, cuda graph: True, gen throughput (token/s): 538.46, #queue-req: 0, [2025-11-30 12:25:12] Decode batch, #running-req: 1, #token: 571, token usage: 0.00, cuda graph: True, gen throughput (token/s): 537.65, #queue-req: 0, [2025-11-30 12:25:12] INFO:     10.86.2.251:55028 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:30] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 313, token usage: 0.00, #running-req: 0, #queue-req: 0, [2025-11-30 12:26:30] Decode batch, #running-req: 1, #token: 319, token usage: 0.00, cuda graph: True, gen throughput (token/s): 0.51, #queue-req: 0, [2025-11-30 12:26:30] Decode batch, #running-req: 1, #token: 359, token usage: 0.00, cuda graph: True, gen throughput (token/s): 529.02, #queue-req: 0, [2025-11-30 12:26:30] Decode batch, #running-req: 1, #token: 399, token usage: 0.00, cuda graph: True, gen throughput (token/s): 535.38, #queue-req: 0, [2025-11-30 12:26:30] Decode batch, #running-req: 1, #token: 439, token usage: 0.00, cuda graph: True, gen throughput (token/s): 537.51, #queue-req: 0, [2025-11-30 12:26:30] Decode batch, #running-req: 1, #token: 479, token usage: 0.00, cuda graph: True, gen throughput (token/s): 538.62, #queue-req: 0, [2025-11-30 12:26:30] Decode batch, #running-req: 1, #token: 519, token usage: 0.00, cuda graph: True, gen throughput (token/s): 537.42, #queue-req: 0, [2025-11-30 12:26:30] INFO:     10.86.2.251:37170 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:37] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 313, token usage: 0.00, #running-req: 0, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 314, token usage: 0.00, #running-req: 1, #queue-req: 0, [2025-11-30 12:26:37] Decode batch, #running-req: 2, #token: 361, token usage: 0.00, cuda graph: True, gen throughput (token/s): 6.15, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 314, token usage: 0.00, #running-req: 2, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 314, token usage: 0.00, #running-req: 3, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 3, #new-token: 3, #cached-token: 942, token usage: 0.00, #running-req: 4, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 3, #new-token: 3, #cached-token: 943, token usage: 0.00, #running-req: 7, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 2, #new-token: 2, #cached-token: 630, token usage: 0.00, #running-req: 10, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 4, #new-token: 4, #cached-token: 1263, token usage: 0.00, #running-req: 12, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 3, #new-token: 3, #cached-token: 948, token usage: 0.00, #running-req: 16, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 3, #new-token: 3, #cached-token: 954, token usage: 0.00, #running-req: 19, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 4, #new-token: 4, #cached-token: 1272, token usage: 0.00, #running-req: 22, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 2, #new-token: 2, #cached-token: 636, token usage: 0.00, #running-req: 26, #queue-req: 0, [2025-11-30 12:26:37] Prefill batch, #new-seq: 2, #new-token: 2, #cached-token: 636, token usage: 0.00, #running-req: 28, #queue-req: 0, [2025-11-30 12:26:37] Decode batch, #running-req: 30, #token: 2021, token usage: 0.00, cuda graph: True, gen throughput (token/s): 4336.92, #queue-req: 0, [2025-11-30 12:26:37] Decode batch, #running-req: 30, #token: 3221, token usage: 0.01, cuda graph: True, gen throughput (token/s): 11959.39, #queue-req: 0, [2025-11-30 12:26:37] Decode batch, #running-req: 30, #token: 4421, token usage: 0.01, cuda graph: True, gen throughput (token/s): 11708.36, #queue-req: 0, [2025-11-30 12:26:38] Decode batch, #running-req: 30, #token: 5621, token usage: 0.01, cuda graph: True, gen throughput (token/s): 11580.60, #queue-req: 0, [2025-11-30 12:26:38] INFO:     10.86.2.251:44692 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] INFO:     10.86.2.251:44678 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] Decode batch, #running-req: 28, #token: 6375, token usage: 0.01, cuda graph: True, gen throughput (token/s): 11296.51, #queue-req: 0, [2025-11-30 12:26:38] INFO:     10.86.2.251:44830 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] INFO:     10.86.2.251:44770 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] Decode batch, #running-req: 26, #token: 7023, token usage: 0.01, cuda graph: True, gen throughput (token/s): 9828.95, #queue-req: 0, [2025-11-30 12:26:38] INFO:     10.86.2.251:44708 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] INFO:     10.86.2.251:44892 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] INFO:     10.86.2.251:44926 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] Decode batch, #running-req: 23, #token: 7155, token usage: 0.01, cuda graph: True, gen throughput (token/s): 9178.79, #queue-req: 0, [2025-11-30 12:26:38] INFO:     10.86.2.251:44908 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] Decode batch, #running-req: 22, #token: 7759, token usage: 0.01, cuda graph: True, gen throughput (token/s): 8645.30, #queue-req: 0, [2025-11-30 12:26:38] Decode batch, #running-req: 22, #token: 8639, token usage: 0.01, cuda graph: True, gen throughput (token/s): 8113.30, #queue-req: 0, [2025-11-30 12:26:38] INFO:     10.86.2.251:44792 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] Decode batch, #running-req: 22, #token: 9123, token usage: 0.02, cuda graph: True, gen throughput (token/s): 7843.11, #queue-req: 0, [2025-11-30 12:26:38] INFO:     10.86.2.251:44878 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] INFO:     10.86.2.251:44840 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] INFO:     10.86.2.251:44858 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] Decode batch, #running-req: 18, #token: 8614, token usage: 0.01, cuda graph: True, gen throughput (token/s): 6806.24, #queue-req: 0, [2025-11-30 12:26:38] INFO:     10.86.2.251:44844 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] Decode batch, #running-req: 17, #token: 8858, token usage: 0.01, cuda graph: True, gen throughput (token/s): 6156.36, #queue-req: 0, [2025-11-30 12:26:38] INFO:     10.86.2.251:44720 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] INFO:     10.86.2.251:44912 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:38] Decode batch, #running-req: 15, #token: 8464, token usage: 0.01, cuda graph: True, gen throughput (token/s): 6201.08, #queue-req: 0, [2025-11-30 12:26:39] INFO:     10.86.2.251:44726 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:39] INFO:     10.86.2.251:44764 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:39] INFO:     10.86.2.251:44888 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:39] Decode batch, #running-req: 12, #token: 7276, token usage: 0.01, cuda graph: True, gen throughput (token/s): 5544.58, #queue-req: 0, [2025-11-30 12:26:39] Decode batch, #running-req: 12, #token: 7756, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4661.07, #queue-req: 0, [2025-11-30 12:26:39] INFO:     10.86.2.251:44820 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:39] Decode batch, #running-req: 11, #token: 7600, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4592.00, #queue-req: 0, [2025-11-30 12:26:39] Decode batch, #running-req: 11, #token: 8040, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4317.14, #queue-req: 0, [2025-11-30 12:26:39] INFO:     10.86.2.251:44806 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:39] INFO:     10.86.2.251:44784 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:39] Decode batch, #running-req: 9, #token: 6968, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4168.81, #queue-req: 0, [2025-11-30 12:26:39] Decode batch, #running-req: 9, #token: 7328, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3482.42, #queue-req: 0, [2025-11-30 12:26:39] INFO:     10.86.2.251:44750 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:39] Decode batch, #running-req: 8, #token: 6892, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3461.23, #queue-req: 0, [2025-11-30 12:26:39] Decode batch, #running-req: 8, #token: 7212, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3431.64, #queue-req: 0, [2025-11-30 12:26:39] INFO:     10.86.2.251:44914 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:39] Decode batch, #running-req: 7, #token: 6656, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3216.16, #queue-req: 0, [2025-11-30 12:26:39] Decode batch, #running-req: 7, #token: 6936, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3042.85, #queue-req: 0, [2025-11-30 12:26:40] Decode batch, #running-req: 7, #token: 7216, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3014.36, #queue-req: 0, [2025-11-30 12:26:40] INFO:     10.86.2.251:44874 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:40] Decode batch, #running-req: 6, #token: 6500, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2697.09, #queue-req: 0, [2025-11-30 12:26:40] Decode batch, #running-req: 6, #token: 6740, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2458.18, #queue-req: 0, [2025-11-30 12:26:40] INFO:     10.86.2.251:44862 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:40] Decode batch, #running-req: 5, #token: 5862, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2284.62, #queue-req: 0, [2025-11-30 12:26:40] Decode batch, #running-req: 5, #token: 6062, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2213.08, #queue-req: 0, [2025-11-30 12:26:40] Decode batch, #running-req: 5, #token: 6262, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2209.57, #queue-req: 0, [2025-11-30 12:26:40] Decode batch, #running-req: 5, #token: 6462, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2206.49, #queue-req: 0, [2025-11-30 12:26:40] INFO:     10.86.2.251:44920 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:40] Decode batch, #running-req: 4, #token: 5384, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2082.59, #queue-req: 0, [2025-11-30 12:26:40] Decode batch, #running-req: 4, #token: 5544, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1814.35, #queue-req: 0, [2025-11-30 12:26:40] Decode batch, #running-req: 4, #token: 5704, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1808.25, #queue-req: 0, [2025-11-30 12:26:40] Decode batch, #running-req: 4, #token: 5864, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1804.46, #queue-req: 0, [2025-11-30 12:26:41] INFO:     10.86.2.251:44850 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:41] Decode batch, #running-req: 3, #token: 4587, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1662.57, #queue-req: 0, [2025-11-30 12:26:41] Decode batch, #running-req: 3, #token: 4707, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1379.24, #queue-req: 0, [2025-11-30 12:26:41] Decode batch, #running-req: 3, #token: 4827, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1378.12, #queue-req: 0, [2025-11-30 12:26:41] Decode batch, #running-req: 3, #token: 4947, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1366.61, #queue-req: 0, [2025-11-30 12:26:41] INFO:     10.86.2.251:44734 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:41] Decode batch, #running-req: 3, #token: 3472, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1355.30, #queue-req: 0, [2025-11-30 12:26:41] Decode batch, #running-req: 2, #token: 3552, token usage: 0.01, cuda graph: True, gen throughput (token/s): 947.64, #queue-req: 0, [2025-11-30 12:26:41] Decode batch, #running-req: 2, #token: 3632, token usage: 0.01, cuda graph: True, gen throughput (token/s): 941.13, #queue-req: 0, [2025-11-30 12:26:41] Decode batch, #running-req: 2, #token: 3712, token usage: 0.01, cuda graph: True, gen throughput (token/s): 935.15, #queue-req: 0, [2025-11-30 12:26:41] INFO:     10.86.2.251:44890 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:26:41] Decode batch, #running-req: 1, #token: 2033, token usage: 0.00, cuda graph: True, gen throughput (token/s): 585.02, #queue-req: 0, [2025-11-30 12:26:41] Decode batch, #running-req: 1, #token: 2073, token usage: 0.00, cuda graph: True, gen throughput (token/s): 516.12, #queue-req: 0, [2025-11-30 12:26:41] Decode batch, #running-req: 1, #token: 2113, token usage: 0.00, cuda graph: True, gen throughput (token/s): 515.79, #queue-req: 0, [2025-11-30 12:26:41] Decode batch, #running-req: 1, #token: 2153, token usage: 0.00, cuda graph: True, gen throughput (token/s): 514.99, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2193, token usage: 0.00, cuda graph: True, gen throughput (token/s): 513.74, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2233, token usage: 0.00, cuda graph: True, gen throughput (token/s): 513.50, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2273, token usage: 0.00, cuda graph: True, gen throughput (token/s): 513.50, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2313, token usage: 0.00, cuda graph: True, gen throughput (token/s): 512.51, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2353, token usage: 0.00, cuda graph: True, gen throughput (token/s): 512.20, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2393, token usage: 0.00, cuda graph: True, gen throughput (token/s): 512.13, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2433, token usage: 0.00, cuda graph: True, gen throughput (token/s): 511.53, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2473, token usage: 0.00, cuda graph: True, gen throughput (token/s): 510.73, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2513, token usage: 0.00, cuda graph: True, gen throughput (token/s): 510.09, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2553, token usage: 0.00, cuda graph: True, gen throughput (token/s): 509.11, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2593, token usage: 0.00, cuda graph: True, gen throughput (token/s): 508.62, #queue-req: 0, [2025-11-30 12:26:42] Decode batch, #running-req: 1, #token: 2633, token usage: 0.00, cuda graph: True, gen throughput (token/s): 508.44, #queue-req: 0, [2025-11-30 12:26:43] Decode batch, #running-req: 1, #token: 2673, token usage: 0.00, cuda graph: True, gen throughput (token/s): 507.87, #queue-req: 0, [2025-11-30 12:26:43] Decode batch, #running-req: 1, #token: 2713, token usage: 0.00, cuda graph: True, gen throughput (token/s): 506.70, #queue-req: 0, [2025-11-30 12:26:43] Decode batch, #running-req: 1, #token: 2753, token usage: 0.00, cuda graph: True, gen throughput (token/s): 506.38, #queue-req: 0, [2025-11-30 12:26:43] Decode batch, #running-req: 1, #token: 2793, token usage: 0.00, cuda graph: True, gen throughput (token/s): 506.59, #queue-req: 0, [2025-11-30 12:26:43] Decode batch, #running-req: 1, #token: 2833, token usage: 0.00, cuda graph: True, gen throughput (token/s): 505.79, #queue-req: 0, [2025-11-30 12:26:43] Decode batch, #running-req: 1, #token: 2873, token usage: 0.00, cuda graph: True, gen throughput (token/s): 505.79, #queue-req: 0, [2025-11-30 12:26:43] Decode batch, #running-req: 1, #token: 2913, token usage: 0.00, cuda graph: True, gen throughput (token/s): 504.96, #queue-req: 0, [2025-11-30 12:26:43] Decode batch, #running-req: 1, #token: 2953, token usage: 0.00, cuda graph: True, gen throughput (token/s): 502.41, #queue-req: 0, [2025-11-30 12:26:43] INFO:     10.86.2.251:44860 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 12:30:01] SIGTERM received. signum=None frame=None. Draining requests and shutting down...[2025-11-30 12:30:02] Gracefully exiting... Remaining number of requests 0. Remaining requests remaining_rids=[].Killed(crowd-pilot) [franz.srambical@hai001.haicore.berlin:~/crowd-pilot-extension] $ python3 -m sglang.launch_server --model-path $model_id --host 0.0.0.02025-11-30 12:45:10.467909: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.^CTraceback (most recent call last):  File "<frozen runpy>", line 198, in _run_module_as_main  File "<frozen runpy>", line 88, in _run_code  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/sglang/launch_server.py", line 7, in <module>    from sglang.srt.server_args import prepare_server_args  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/sglang/srt/server_args.py", line 61, in <module>    from sglang.srt.utils.hf_transformers_utils import check_gguf_file, get_config  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/sglang/srt/utils/hf_transformers_utils.py", line 26, in <module>    from transformers import (    ...<8 lines>...    )  File "<frozen importlib._bootstrap>", line 1412, in _handle_fromlist  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/transformers/utils/import_utils.py", line 2317, in __getattr__    module = self._get_module(self._class_to_module[name])  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/transformers/utils/import_utils.py", line 2345, in _get_module    return importlib.import_module("." + module_name, self.__name__)           ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^  File "/home/franz.srambical/.local/share/uv/python/cpython-3.13.5-linux-x86_64-gnu/lib/python3.13/importlib/__init__.py", line 88, in import_module    return _bootstrap._gcd_import(name[level:], package, level)           ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/transformers/models/auto/processing_auto.py", line 27, in <module>    from ...image_processing_utils import ImageProcessingMixin  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/transformers/image_processing_utils.py", line 22, in <module>    from .image_transforms import center_crop, normalize, rescale  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/transformers/image_transforms.py", line 48, in <module>    import tensorflow as tf  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/tensorflow/__init__.py", line 40, in <module>    from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow  # pylint: disable=unused-import    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/tensorflow/python/pywrap_tensorflow.py", line 37, in <module>    self_check.preload_check()    ~~~~~~~~~~~~~~~~~~~~~~~~^^  File "/fast/home/franz.srambical/crowd-pilot/.venv/lib/python3.13/site-packages/tensorflow/python/platform/self_check.py", line 63, in preload_check    from tensorflow.python.platform import _pywrap_cpu_feature_guardKeyboardInterrupt^C(crowd-pilot) [franz.srambical@hai001.haicore.berlin:~/crowd-pilot-extension] $ exitexitsrun: error: hai001: task 0: Exited with exit code 130salloc: Relinquishing job allocation 35151salloc: Job allocation 35151 has been revoked.