[terminal_command] TERMINAL ()
[2025-11-30 11:37:35] Decode batch, #running-req: 1, #token: 2199, token usage: 0.00, cuda graph: True, gen throughput (token/s): 511.70, #queue-req: 0, [2025-11-30 11:37:36] INFO:     10.86.2.251:40808 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:54] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 318, token usage: 0.00, #running-req: 0, #queue-req: 0, [2025-11-30 11:37:54] Prefill batch, #new-seq: 2, #new-token: 84, #cached-token: 554, token usage: 0.00, #running-req: 1, #queue-req: 0, [2025-11-30 11:37:54] Prefill batch, #new-seq: 4, #new-token: 40, #cached-token: 1231, token usage: 0.00, #running-req: 3, #queue-req: 0, [2025-11-30 11:37:54] Prefill batch, #new-seq: 3, #new-token: 21, #cached-token: 924, token usage: 0.00, #running-req: 7, #queue-req: 0, [2025-11-30 11:37:54] Decode batch, #running-req: 10, #token: 727, token usage: 0.00, cuda graph: True, gen throughput (token/s): 9.89, #queue-req: 0, [2025-11-30 11:37:54] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 313, token usage: 0.00, #running-req: 10, #queue-req: 0, [2025-11-30 11:37:54] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 314, token usage: 0.00, #running-req: 11, #queue-req: 0, [2025-11-30 11:37:54] Prefill batch, #new-seq: 2, #new-token: 2, #cached-token: 628, token usage: 0.00, #running-req: 12, #queue-req: 0, [2025-11-30 11:37:54] Decode batch, #running-req: 14, #token: 1290, token usage: 0.00, cuda graph: True, gen throughput (token/s): 4218.38, #queue-req: 0, [2025-11-30 11:37:54] Prefill batch, #new-seq: 2, #new-token: 2, #cached-token: 628, token usage: 0.00, #running-req: 14, #queue-req: 0, [2025-11-30 11:37:54] Decode batch, #running-req: 16, #token: 1927, token usage: 0.00, cuda graph: True, gen throughput (token/s): 6099.60, #queue-req: 0, [2025-11-30 11:37:54] Decode batch, #running-req: 16, #token: 2567, token usage: 0.00, cuda graph: True, gen throughput (token/s): 7187.37, #queue-req: 0, [2025-11-30 11:37:54] Decode batch, #running-req: 16, #token: 3207, token usage: 0.01, cuda graph: True, gen throughput (token/s): 7087.41, #queue-req: 0, [2025-11-30 11:37:54] INFO:     10.86.2.251:57724 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:55] INFO:     10.86.2.251:57778 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:55] INFO:     10.86.2.251:57692 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:55] INFO:     10.86.2.251:57772 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:55] Decode batch, #running-req: 12, #token: 2691, token usage: 0.00, cuda graph: True, gen throughput (token/s): 6243.55, #queue-req: 0, [2025-11-30 11:37:55] INFO:     10.86.2.251:57728 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:55] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 314, token usage: 0.00, #running-req: 11, #queue-req: 0, [2025-11-30 11:37:55] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 314, token usage: 0.00, #running-req: 12, #queue-req: 0, [2025-11-30 11:37:55] INFO:     10.86.2.251:57764 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:55] INFO:     10.86.2.251:57786 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:55] Decode batch, #running-req: 11, #token: 2719, token usage: 0.00, cuda graph: True, gen throughput (token/s): 4205.10, #queue-req: 0, [2025-11-30 11:37:55] INFO:     10.86.2.251:57812 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:55] Decode batch, #running-req: 10, #token: 2940, token usage: 0.00, cuda graph: True, gen throughput (token/s): 4285.31, #queue-req: 0, [2025-11-30 11:37:55] INFO:     10.86.2.251:57688 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:55] Decode batch, #running-req: 9, #token: 2961, token usage: 0.00, cuda graph: True, gen throughput (token/s): 4227.10, #queue-req: 0, [2025-11-30 11:37:55] Decode batch, #running-req: 9, #token: 3321, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3970.99, #queue-req: 0, [2025-11-30 11:37:55] Decode batch, #running-req: 9, #token: 3681, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3903.86, #queue-req: 0, [2025-11-30 11:37:55] Prefill batch, #new-seq: 1, #new-token: 19, #cached-token: 295, token usage: 0.01, #running-req: 9, #queue-req: 0, [2025-11-30 11:37:55] Prefill batch, #new-seq: 1, #new-token: 1, #cached-token: 313, token usage: 0.01, #running-req: 10, #queue-req: 0, [2025-11-30 11:37:55] Decode batch, #running-req: 11, #token: 4106, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3234.89, #queue-req: 0, [2025-11-30 11:37:55] Decode batch, #running-req: 11, #token: 4546, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4521.55, #queue-req: 0, [2025-11-30 11:37:55] INFO:     10.86.2.251:57814 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:55] Decode batch, #running-req: 10, #token: 4527, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4327.21, #queue-req: 0, [2025-11-30 11:37:55] Decode batch, #running-req: 10, #token: 4927, token usage: 0.01, cuda graph: True, gen throughput (token/s): 4168.36, #queue-req: 0, [2025-11-30 11:37:55] INFO:     10.86.2.251:57858 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:56] Decode batch, #running-req: 9, #token: 5125, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3972.31, #queue-req: 0, [2025-11-30 11:37:56] Decode batch, #running-req: 9, #token: 5485, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3702.26, #queue-req: 0, [2025-11-30 11:37:56] INFO:     10.86.2.251:57802 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:56] Decode batch, #running-req: 8, #token: 5157, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3592.36, #queue-req: 0, [2025-11-30 11:37:56] Decode batch, #running-req: 8, #token: 5477, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3532.38, #queue-req: 0, [2025-11-30 11:37:56] Decode batch, #running-req: 8, #token: 5797, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3500.69, #queue-req: 0, [2025-11-30 11:37:56] Decode batch, #running-req: 8, #token: 6117, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3485.28, #queue-req: 0, [2025-11-30 11:37:56] INFO:     10.86.2.251:57818 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:56] Decode batch, #running-req: 7, #token: 5768, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3365.68, #queue-req: 0, [2025-11-30 11:37:56] Decode batch, #running-req: 7, #token: 6048, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3065.42, #queue-req: 0, [2025-11-30 11:37:56] Decode batch, #running-req: 7, #token: 6328, token usage: 0.01, cuda graph: True, gen throughput (token/s): 3049.83, #queue-req: 0, [2025-11-30 11:37:56] INFO:     10.86.2.251:57740 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:56] INFO:     10.86.2.251:57672 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:56] Decode batch, #running-req: 5, #token: 4613, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2803.99, #queue-req: 0, [2025-11-30 11:37:56] Decode batch, #running-req: 5, #token: 4231, token usage: 0.01, cuda graph: True, gen throughput (token/s): 2225.82, #queue-req: 0, [2025-11-30 11:37:56] INFO:     10.86.2.251:57846 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:57] Decode batch, #running-req: 4, #token: 4390, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1866.82, #queue-req: 0, [2025-11-30 11:37:57] Decode batch, #running-req: 4, #token: 4550, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1847.37, #queue-req: 0, [2025-11-30 11:37:57] INFO:     10.86.2.251:57756 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:57] Decode batch, #running-req: 3, #token: 3532, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1643.18, #queue-req: 0, [2025-11-30 11:37:57] Decode batch, #running-req: 3, #token: 3652, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1401.17, #queue-req: 0, [2025-11-30 11:37:57] Decode batch, #running-req: 3, #token: 3772, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1404.23, #queue-req: 0, [2025-11-30 11:37:57] Decode batch, #running-req: 3, #token: 3892, token usage: 0.01, cuda graph: True, gen throughput (token/s): 1398.07, #queue-req: 0, [2025-11-30 11:37:57] INFO:     10.86.2.251:57830 - "POST /v1/chat/completions HTTP/1.1" 200 OK[2025-11-30 11:37:57] Decode batch, #running-req: 2, #token: 2903, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1332.51, #queue-req: 0, [2025-11-30 11:37:57] Decode batch, #running-req: 2, #token: 2983, token usage: 0.00, cuda graph: True, gen throughput (token/s): 955.42, #queue-req: 0, [2025-11-30 11:37:57] Decode batch, #running-req: 2, #token: 3063, token usage: 0.01, cuda graph: True, gen throughput (token/s): 952.06, #queue-req: 0, [2025-11-30 11:37:57] Decode batch, #running-req: 2, #token: 3143, token usage: 0.01, cuda graph: True, gen throughput (token/s): 949.95, #queue-req: 0, [2025-11-30 11:37:57] Decode batch, #running-req: 2, #token: 3223, token usage: 0.01, cuda graph: True, gen throughput (token/s): 949.63, #queue-req: 0, [2025-11-30 11:37:57] Decode batch, #running-req: 2, #token: 3303, token usage: 0.01, cuda graph: True, gen throughput (token/s): 949.12, #queue-req: 0, [2025-11-30 11:37:58] Decode batch, #running-req: 2, #token: 3383, token usage: 0.01, cuda graph: True, gen throughput (token/s): 947.57, #queue-req: 0, [2025-11-30 11:37:58] Decode batch, #running-req: 2, #token: 3463, token usage: 0.01, cuda graph: True, gen throughput (token/s): 943.19, #queue-req: 0, [2025-11-30 11:37:58] Decode batch, #running-req: 2, #token: 3543, token usage: 0.01, cuda graph: True, gen throughput (token/s): 943.42, #queue-req: 0, [2025-11-30 12:00:01] SIGTERM received. signum=None frame=None. Draining requests and shutting down...[2025-11-30 12:00:02] Gracefully exiting... Remaining number of requests 0. Remaining requests remaining_rids=[].Killed(crowd-pilot) [franz.srambical@hai001.haicore.berlin:~/crowd-pilot-extension] $ squeue  JOBID       USER PARTITION NODES  CPUS ST         SUBMIT_TIME          START_TIME     TIME TIME_LIMIT NODELIST(REASON)  35150 franz.sram interacti     1    20  R 2025-11-30T11:32:41 2025-11-30T11:32:41    35:51 1-00:00:00 hai001  35149 alfred.ngu interacti     1     8  R 2025-11-30T11:30:10 2025-11-30T11:30:10    38:22    2:00:00 hai003  35144   xiao.liu interacti     1   128  R 2025-11-30T06:03:00 2025-11-30T06:03:00  6:05:32   23:59:00 hai006  35143   xiao.liu interacti     1   128  R 2025-11-29T20:48:57 2025-11-29T20:48:57 15:19:35   23:59:00 hai005  35130   xiao.liu  standard     1   128  R 2025-11-29T08:31:20 2025-11-29T13:48:24 22:20:08   23:59:00 hai004(crowd-pilot) [franz.srambical@hai001.haicore.berlin:~/crowd-pilot-extension] $ scancel 35150salloc: Job allocation 35150 has been revoked.                                              srun: Job step aborted: Waiting up to 32 seconds for job step to finish.(crowd-pilot) [franz.srambical@hai001.haicore.berlin:~/crowd-pilot-extension] $ [2025-11-30T12:08:45.442] error: *** STEP 35150.interactive ON hai001 CANCELLED AT 2025-11-30T12:08:45 DUE to SIGNAL Terminated ***^C(crowd-pilot) [franz.srambical@hai001.haicore.berlin:~/crowd-pilot-extension] $ ^C(crowd-pilot) [franz.srambical@hai001.haicore.berlin:~/crowd-pilot-extension] $ squeue  JOBID       USER PARTITION NODES  CPUS ST         SUBMIT_TIME          START_TIME     TIME TIME_LIMIT NODELIST(REASON)  35150 franz.sram interacti     1    20 CG 2025-11-30T11:32:41 2025-11-30T11:32:41    36:04 1-00:00:00 hai001  35149 alfred.ngu interacti     1     8  R 2025-11-30T11:30:10 2025-11-30T11:30:10    38:54    2:00:00 hai003  35144   xiao.liu interacti     1   128  R 2025-11-30T06:03:00 2025-11-30T06:03:00  6:06:04   23:59:00 hai006  35143   xiao.liu interacti     1   128  R 2025-11-29T20:48:57 2025-11-29T20:48:57 15:20:07   23:59:00 hai005  35130   xiao.liu  standard     1   128  R 2025-11-29T08:31:20 2025-11-29T13:48:24 22:20:40   23:59:00 hai004(crowd-pilot) [franz.srambical@hai001.haicore.berlin:~/crowd-pilot-extension] $ srun: error: hai001: task 0: Killed