[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_franz/maskgit/dynamics-cotraining/train_dynamics_maskgit_8_node_3422132.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_franz/maskgit/dynamics-cotraining/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_franz/maskgit/dynamics-cotraining/%x_%j.log
#SBATCH --job-name=train_dynamics_maskgit_8_node
#SBATCH --requeue
#SBATCH --reservation=llmtum
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi



# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/holiday/maskgit/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401

env | grep SLURM

srun python train_dynamics.py \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=160 \
    --init_lr=0 \
    --darkness_threshold=50 \
    --dyna_type=maskgit \
    --num_latent_actions=6 \
    --max_lr=8e-5 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=dynamics-maskgit-8-node-darkness-filter-$slurm_job_id \
    --tags dynamics maskgit 8-node post-launch-main darkness-filter \
    --entity instant-uv \
    --project jafar \
    --dyna_dim=1024 \
    --dyna_num_blocks=16 \
    --dyna_num_heads=16 \
    --dyna_ffn_dim=4096 \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
SLURM_JOB_USER=tum_dbd0378
SLURM_TASKS_PER_NODE=4(x8)
SLURM_JOB_UID=996262
SLURM_TASK_PID=17640
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar
SLURMD_NODENAME=hkn0530
SLURM_JOB_START_TIME=1755170692
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1755343492
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x8)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=8
SLURM_JOBID=3422132
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_JOB_RESERVATION=llmtum
SLURM_NTASKS=32
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e11.hkn0530
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn[0530-0534,0536,0701,0715]
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=32
SLURM_NNODES=8
SLURM_SUBMIT_HOST=hkn1990.localdomain
SLURM_JOB_ID=3422132
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dynamics_maskgit_8_node
SLURM_NTASKS_PER_NODE=4
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0530-0534,0536,0701,0715]
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
2025-08-14 13:25:47.802222: W external/xla/xla/service/platform_util.cc:220] unable to create StreamExecutor for CUDA:3: CUDA error: Failed call to cuDeviceGet: CUDA_ERROR_INVALID_DEVICE: invalid device ordinal
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: INTERNAL: no supported devices found for platform CUDA

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/train_dynamics.py", line 158, in <module>
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': INTERNAL: no supported devices found for platform CUDA (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: INTERNAL: Getting local topologies failed: Error 1: GetKeyValue() timed out with key: cuda:local_topology/cuda/19 and duration: 2m

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/train_dynamics.py", line 158, in <module>
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': INTERNAL: Getting local topologies failed: Error 1: GetKeyValue() timed out with key: cuda:local_topology/cuda/19 and duration: 2m (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: DEADLINE_EXCEEDED: GetKeyValue() timed out with key: cuda:global_topology/cuda and duration: 5m

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/train_dynamics.py", line 158, in <module>
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': DEADLINE_EXCEEDED: GetKeyValue() timed out with key: cuda:global_topology/cuda and duration: 5m (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: DEADLINE_EXCEEDED: GetKeyValue() timed out with key: cuda:global_topology/cuda and duration: 5m

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/train_dynamics.py", line 158, in <module>
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': DEADLINE_EXCEEDED: GetKeyValue() timed out with key: cuda:global_topology/cuda and duration: 5m (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: DEADLINE_EXCEEDED: GetKeyValue() timed out with key: cuda:global_topology/cuda and duration: 5m

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/train_dynamics.py", line 158, in <module>
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_dbd0378/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': DEADLINE_EXCEEDED: GetKeyValue() timed out with key: cuda:global_topology/cuda and duration: 5m (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
2025-08-14 13:30:48.404038: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service.cc:1798] Shutdown barrier in coordination service has failed:
DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']
This suggests that the workers are out of sync. Either at least one worker (a) crashed early due to program error or scheduler events (e.g. preemption, eviction), (b) was too fast in its execution, or (c) too slow / hanging. Check the logs (both the program and scheduler events) for an earlier error to identify the root cause.
2025-08-14 13:30:48.404124: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service.cc:1839] Use error polling to propagate the following error to all tasks: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296'] [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.404702: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.404606: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.404610: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.404778: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405016: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.404842: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.404891: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.404932: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405085: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.404913: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405092: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.404916: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405030: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405001: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.405055: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405142: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405049: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405214: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405066: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405307: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405334: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.405487: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.405255: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405406: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405332: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.405365: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405622: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405408: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.405451: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405527: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405438: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405537: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405620: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405587: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405776: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405493: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.405770: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405462: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405773: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.405592: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405765: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.405437: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405935: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405994: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405866: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.406066: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.405889: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

2025-08-14 13:30:48.406077: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.404956: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.404986: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405212: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.405510: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.421574: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.421570: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.421590: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.421569: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.512644: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.512567: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.512644: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.512749: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-14 13:30:48.513476: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.513541: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.513499: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-14 13:30:48.513545: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::3040337475261767296::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 5/32.
The first task at the barrier: /job:jax_worker/replica:0/task:19. Some timed out task names:
/job:jax_worker/replica:0/task:20
/job:jax_worker/replica:0/task:30
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::3040337475261767296']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
srun: error: hkn0534: task 19: Aborted (core dumped)
srun: error: hkn0534: tasks 16,18: Aborted (core dumped)
srun: error: hkn0533: tasks 12,14: Aborted (core dumped)
srun: error: hkn0532: tasks 9,11: Aborted (core dumped)
srun: error: hkn0530: tasks 0,2: Aborted (core dumped)
srun: error: hkn0531: tasks 4,7: Aborted (core dumped)
srun: error: hkn0536: tasks 21-22: Aborted (core dumped)
srun: error: hkn0701: tasks 26-27: Aborted (core dumped)
srun: error: hkn0715: tasks 30-31: Aborted (core dumped)
srun: error: hkn0534: task 17: Aborted (core dumped)
srun: error: hkn0533: task 15: Aborted (core dumped)
srun: error: hkn0532: task 8: Aborted (core dumped)
srun: error: hkn0531: task 6: Aborted (core dumped)
srun: error: hkn0530: task 3: Aborted (core dumped)
srun: error: hkn0536: task 23: Aborted (core dumped)
srun: error: hkn0701: task 25: Aborted (core dumped)
srun: error: hkn0715: task 28: Aborted (core dumped)
srun: error: hkn0533: task 13: Aborted (core dumped)
srun: error: hkn0532: task 10: Aborted (core dumped)
srun: error: hkn0530: task 1: Aborted (core dumped)
srun: error: hkn0531: task 5: Aborted (core dumped)
srun: error: hkn0536: task 20: Aborted (core dumped)
srun: error: hkn0701: task 24: Aborted (core dumped)
srun: error: hkn0715: task 29: Aborted (core dumped)

============================= JOB FEEDBACK =============================

Job ID: 3422132
Cluster: hk
User/Group: tum_dbd0378/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 134)
Partition: accelerated
Nodes: 8
Cores per node: 24
Nodelist: hkn[0530-0534,0536,0701,0715]
CPU Utilized: 00:02:54
CPU Efficiency: 0.25% of 19:02:24 core-walltime
Job Wall-clock time: 00:05:57
Starttime: Thu Aug 14 13:24:52 2025
Endtime: Thu Aug 14 13:30:49 2025
Memory Utilized: 12.67 GB (estimated maximum)
Memory Efficiency: 0.00% of 0.00 MB (0.00 MB/node)
Energy Consumed: 1339227 Joule / 372.0075 Watthours
Average node power draw: 3751.33613445378 Watt