[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/lam/train_lam_minecraft_8node_darkness_filter_37M_3457969.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/lam/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/lam/%x_%j.log
#SBATCH --job-name=train_lam_minecraft_8node_darkness_filter_37M
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
# slurm_job_id=$SLURM_JOB_ID
slurm_job_id=3454953

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/lam/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --save_ckpt \
    --restore_ckpt \
    --wandb_id $slurm_job_id \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=160 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --darkness_threshold=50 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=lam-minecraft-8-node-darkness-filter-37M-$slurm_job_id \
    --tags lam minecraft 8-node darkness-filter 37M \
    --entity instant-uv \
    --project jafar \
    --num_latents=100 \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
/var/spool/slurmd/job3457969/slurm_script: line 38: .venv/bin/activate: No such file or directory
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x8)
SLURM_JOB_UID=999226
SLURM_TASK_PID=1440498
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs
SLURMD_NODENAME=hkn0404
SLURM_JOB_START_TIME=1756824635
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1756997435
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x8)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=8
SLURM_JOBID=3457969
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=32
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e11.hkn0404
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn[0404,0407-0408,0417,0503,0725,0729,0732]
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=32
SLURM_NNODES=8
SLURM_SUBMIT_HOST=hkn1990.localdomain
SLURM_JOB_ID=3457969
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_lam_minecraft_8node_darkness_filter_37M
SLURM_NTASKS_PER_NODE=4
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0404,0407-0408,0417,0503,0725,0729,0732]
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
2025-09-02 16:51:32.979826: E external/xla/xla/stream_executor/cuda/cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
2025-09-02 16:51:32.979716: E external/xla/xla/stream_executor/cuda/cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
2025-09-02 16:51:32.979963: E external/xla/xla/stream_executor/cuda/cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
2025-09-02 16:51:32.979958: E external/xla/xla/stream_executor/cuda/cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_lam.py", line 153, in <module>
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_lam.py", line 153, in <module>
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_lam.py", line 153, in <module>
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_lam.py", line 153, in <module>
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
2025-09-02 16:56:33.847340: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service.cc:1798] Shutdown barrier in coordination service has failed:
DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']
This suggests that the workers are out of sync. Either at least one worker (a) crashed early due to program error or scheduler events (e.g. preemption, eviction), (b) was too fast in its execution, or (c) too slow / hanging. Check the logs (both the program and scheduler events) for an earlier error to identify the root cause.
2025-09-02 16:56:33.847379: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service.cc:1839] Use error polling to propagate the following error to all tasks: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055'] [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.847562: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.847711: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.847756: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Failed to disconnect from coordination service with status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/ShutdownTask [type.googleapis.com/tensorflow.CoordinationServiceError='']Proceeding with agent shutdown anyway. This is usually caused by an earlier error during execution. Check the logs of (a) this task, (b) the leader (usually slice 0 task 0) and (c) the scheduler (e.g. preemption, eviction) for an earlier error to debug further. [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.847765: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.847814: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.847865: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.848049: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848309: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.848088: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848204: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848026: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.848100: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848254: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848027: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848062: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848225: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848211: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848202: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848085: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848183: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848217: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848280: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848218: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848263: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848329: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848260: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848395: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848441: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848796: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.848399: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848611: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.848439: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848749: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848760: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.848398: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.848833: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849068: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.849200: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849066: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.849198: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.849003: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849035: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.849080: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849229: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.849107: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.849359: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.849266: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849228: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.849209: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.849270: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849223: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

2025-09-02 16:56:33.849264: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849408: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849271: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849221: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849411: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849434: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849435: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.849434: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2947760685903380055::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:1. Some timed out task names:
/job:jax_worker/replica:0/task:13
/job:jax_worker/replica:0/task:20
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2947760685903380055']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-09-02 16:56:33.931086: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.931108: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:{"created":"@1756824993.930994068","description":"Error received from peer ipv4:10.0.1.36:62385","file":"external/com_github_grpc_grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Socket closed","grpc_status":14}
2025-09-02 16:56:33.931120: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-09-02 16:56:33.931151: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:{"created":"@1756824993.931007203","description":"Error received from peer ipv4:10.0.1.36:62385","file":"external/com_github_grpc_grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Socket closed","grpc_status":14}
srun: error: hkn0404: tasks 0-3: Aborted (core dumped)
srun: error: hkn0732: tasks 29-30: Aborted (core dumped)
srun: error: hkn0407: tasks 4,7: Aborted (core dumped)
srun: error: hkn0725: tasks 21,23: Aborted (core dumped)
srun: error: hkn0408: tasks 9-10: Aborted (core dumped)
srun: error: hkn0729: tasks 24-25: Aborted (core dumped)
srun: error: hkn0417: tasks 14-15: Aborted (core dumped)
srun: error: hkn0503: tasks 16,18: Aborted (core dumped)
srun: error: hkn0407: task 6: Aborted (core dumped)
srun: error: hkn0732: task 31: Aborted (core dumped)
srun: error: hkn0725: task 22: Aborted (core dumped)
srun: error: hkn0729: task 26: Aborted (core dumped)
srun: error: hkn0503: task 17: Aborted (core dumped)
srun: error: hkn0408: task 11: Aborted (core dumped)
srun: error: hkn0417: task 12: Aborted (core dumped)
srun: error: hkn0732: task 28: Aborted (core dumped)
srun: error: hkn0725: task 20: Aborted (core dumped)
srun: error: hkn0407: task 5: Aborted (core dumped)
srun: error: hkn0408: task 8: Aborted (core dumped)
srun: error: hkn0417: task 13: Aborted (core dumped)
srun: error: hkn0729: task 27: Aborted (core dumped)
srun: error: hkn0503: task 19: Aborted (core dumped)

============================= JOB FEEDBACK =============================

Job ID: 3457969
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 134)
Partition: accelerated
Nodes: 8
Cores per node: 24
Nodelist: hkn[0404,0407-0408,0417,0503,0725,0729,0732]
CPU Utilized: 00:02:37
CPU Efficiency: 0.23% of 19:08:48 core-walltime
Job Wall-clock time: 00:05:59
Starttime: Tue Sep  2 16:50:35 2025
Endtime: Tue Sep  2 16:56:34 2025
Memory Utilized: 12.17 GB (estimated maximum)
Memory Efficiency: 0.00% of 0.00 MB (0.00 MB/node)
Energy Consumed: 3157333 Joule / 877.036944444444 Watthours
Average node power draw: 8794.79944289694 Watt