[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/train_dyn_yolorun_3333026.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=10:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_dyn_yolorun

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

# array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked
tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

lam_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/train_lam_action_space_scaling_20/3318547/lam_1751657975_200000/
tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/train_tokenizer_batch_size_scaling_16_node/3321526/tokenizer_22000/

env | grep SLURM

srun python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=1.4e-4 \
    --log_image_interval=1000 \
    --log \
    --num_latent_actions=20 \
    --log_checkpoint_interval=1000 \
    --name=dynamics-yolorun-tf-records-$slurm_job_id \
    --tags dynamics yolo-run tf_records \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $tf_records_dir \
    --lam_checkpoint=$lam_ckpt_dir

Note: the module "mpi/openmpi/5.0" cannot be unloaded because it was not
loaded.


Note: the module "devel/cuda/12.4" cannot be unloaded because it was not
loaded.

SLURM_STEP_NUM_TASKS=1
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x2)
SLURM_JOB_UID=999226
SLURM_TASK_PID=2752853
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar
SLURMD_NODENAME=hkn0411
SLURM_JOB_START_TIME=1752076731
SLURM_STEP_NODELIST=hkn0523
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1752112731
SLURM_PMI2_SRUN_PORT=41739
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x2)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=2
SLURM_STEPID=4294967290
SLURM_JOBID=3333026
SLURM_PTY_PORT=33857
SLURM_JOB_QOS=normal
SLURM_LAUNCH_NODE_IPADDR=10.0.7.199
SLURM_PTY_WIN_ROW=35
SLURM_PMI2_PROC_MAPPING=(vector,(0,1,1))
SLURMD_DEBUG=2
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=8
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e12.hkn0411
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SRUN_COMM_HOST=10.0.7.199
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_PTY_WIN_COL=182
SLURM_NODELIST=hkn[0411,0705]
SLURM_SRUN_COMM_PORT=40767
SLURM_STEP_ID=4294967290
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=8
SLURM_NNODES=2
SLURM_SUBMIT_HOST=hkn0523.localdomain
SLURM_JOB_ID=3333026
SLURM_NODEID=0
SLURM_STEP_NUM_NODES=1
SLURM_STEP_TASKS_PER_NODE=1
SLURM_MPI_TYPE=pmi2
SLURM_PMI2_STEP_NODES=hkn0523
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dyn_yolorun
SLURM_NTASKS_PER_NODE=4
SLURM_STEP_LAUNCHER_PORT=40767
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0411,0705]
SLURM_STEP_NUM_TASKS=1
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x2)
SLURM_JOB_UID=999226
SLURM_TASK_PID=2752853
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar
SLURMD_NODENAME=hkn0411
SLURM_JOB_START_TIME=1752076731
SLURM_STEP_NODELIST=hkn0523
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1752112731
SLURM_PMI2_SRUN_PORT=41739
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x2)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=2
SLURM_STEPID=4294967290
SLURM_JOBID=3333026
SLURM_PTY_PORT=33857
SLURM_JOB_QOS=normal
SLURM_LAUNCH_NODE_IPADDR=10.0.7.199
SLURM_PTY_WIN_ROW=35
SLURM_PMI2_PROC_MAPPING=(vector,(0,1,1))
SLURMD_DEBUG=2
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=8
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e12.hkn0411
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SRUN_COMM_HOST=10.0.7.199
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_PTY_WIN_COL=182
SLURM_NODELIST=hkn[0411,0705]
SLURM_SRUN_COMM_PORT=40767
SLURM_STEP_ID=4294967290
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=8
SLURM_NNODES=2
SLURM_SUBMIT_HOST=hkn0523.localdomain
SLURM_JOB_ID=3333026
SLURM_NODEID=0
SLURM_STEP_NUM_NODES=1
SLURM_STEP_TASKS_PER_NODE=1
SLURM_MPI_TYPE=pmi2
SLURM_PMI2_STEP_NODES=hkn0523
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dyn_yolorun
SLURM_NTASKS_PER_NODE=4
SLURM_STEP_LAUNCHER_PORT=40767
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0411,0705]
GpuFreq=control_disabled
GpuFreq=control_disabled
2025-07-09 18:10:15.421340: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-09 18:10:15.421336: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-09 18:10:15.421335: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-09 18:10:15.421342: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1752077415.484725 2752915 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1752077415.484775 2752917 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1752077415.485154 2752916 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1752077415.485061 2752918 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1752077415.490233 2752915 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1752077415.490229 2752916 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1752077415.490235 2752917 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1752077415.490262 2752918 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1752077556.793360 2752915 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793428 2752915 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793430 2752915 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793431 2752915 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793358 2752916 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793440 2752916 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793443 2752916 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793444 2752916 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793405 2752917 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793436 2752917 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793438 2752917 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793440 2752917 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793515 2752918 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793540 2752918 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793542 2752918 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077556.793543 2752918 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
2025-07-09 18:18:36.263921: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-09 18:18:36.263920: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-09 18:18:36.263927: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-09 18:18:36.263924: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1752077916.301789 1436988 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1752077916.301790 1436990 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1752077916.302082 1436991 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1752077916.302131 1436989 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1752077916.311773 1436989 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1752077916.311772 1436991 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1752077916.312391 1436988 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1752077916.312396 1436990 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1752077960.330177 1436989 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330227 1436989 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330229 1436989 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330231 1436989 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330165 1436991 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330227 1436991 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330230 1436991 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330232 1436991 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330176 1436988 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330250 1436988 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330257 1436988 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330262 1436988 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330174 1436990 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330286 1436990 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330294 1436990 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752077960.330299 1436990 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1752078577.526818 2752915 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1752078577.526828 2752916 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1752078577.526827 2752917 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1752078577.526808 2752918 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
2025-07-09 18:34:42.675752: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: DEADLINE_EXCEEDED: Deadline Exceeded

RPC: /tensorflow.CoordinationService/RegisterTask
srun: error: hkn0411: task 1: Aborted (core dumped)
2025-07-09 18:34:56.636242: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: DEADLINE_EXCEEDED: Deadline Exceeded

RPC: /tensorflow.CoordinationService/RegisterTask
2025-07-09 18:34:56.695132: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/RegisterTask:
:{"created":"@1752078896.693018837","description":"Error received from peer ipv4:10.0.1.43:64418","file":"external/com_github_grpc_grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Socket closed","grpc_status":14}
2025-07-09 18:34:56.695356: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/RegisterTask:
:{"created":"@1752078896.693108062","description":"Error received from peer ipv4:10.0.1.43:64418","file":"external/com_github_grpc_grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Socket closed","grpc_status":14}
srun: error: hkn0411: tasks 0,2: Aborted (core dumped)
srun: error: hkn0411: task 3: Aborted (core dumped)
W0000 00:00:1752078912.617580 1436988 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1752078912.617569 1436990 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1752078912.617568 1436991 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1752078912.617703 1436989 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
2025-07-09 18:40:17.695088: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: DEADLINE_EXCEEDED: Deadline Exceeded

RPC: /tensorflow.CoordinationService/RegisterTask
2025-07-09 18:40:17.695275: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: DEADLINE_EXCEEDED: Deadline Exceeded

RPC: /tensorflow.CoordinationService/RegisterTask
2025-07-09 18:40:17.695448: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: DEADLINE_EXCEEDED: Deadline Exceeded

RPC: /tensorflow.CoordinationService/RegisterTask
2025-07-09 18:40:17.695449: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: DEADLINE_EXCEEDED: Deadline Exceeded

RPC: /tensorflow.CoordinationService/RegisterTask
srun: error: hkn0705: task 5: Aborted (core dumped)
srun: error: hkn0705: tasks 4,7: Aborted (core dumped)
srun: error: hkn0705: task 6: Aborted (core dumped)

============================= JOB FEEDBACK =============================

Job ID: 3333026
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 134)
Partition: accelerated
Nodes: 2
Cores per node: 24
Nodelist: hkn[0411,0705]
CPU Utilized: 00:00:59
CPU Efficiency: 0.05% of 1-09:09:36 core-walltime
Job Wall-clock time: 00:41:27
Starttime: Wed Jul  9 17:58:51 2025
Endtime: Wed Jul  9 18:40:18 2025
Memory Utilized: 3.79 GB (estimated maximum)
Memory Efficiency: 0.00% of 0.00 MB (0.00 MB/node)
Energy Consumed: 2334115 Joule / 648.365277777778 Watthours
Average node power draw: 938.526336952151 Watt