[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/maskgit/dynamics-cotraining/train_dynamics_maskgit_8_node_3418834.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/maskgit/dynamics-cotraining/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/maskgit/dynamics-cotraining/%x_%j.log
#SBATCH --job-name=train_dynamics_maskgit_8_node
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi



# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/holiday/maskgit/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401

env | grep SLURM

srun python train_dynamics.py \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=160 \
    --init_lr=0 \
    --darkness_threshold=50 \
    --dyna_type=maskgit \
    --num_latent_actions=100 \
    --max_lr=8e-5 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=dynamics-maskgit-8-node-darkness-filter-$slurm_job_id \
    --tags dynamics maskgit 8-node post-launch-main darkness-filter \
    --entity instant-uv \
    --project jafar \
    --dyna_dim=1024 \
    --dyna_num_blocks=16 \
    --dyna_num_heads=16 \
    --dyna_ffn_dim=4096 \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
/var/spool/slurmd/job3418834/slurm_script: line 42: .venv/bin/activate: No such file or directory
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x8)
SLURM_JOB_UID=999226
SLURM_TASK_PID=2792684
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs
SLURMD_NODENAME=hkn0531
SLURM_JOB_START_TIME=1755086531
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1755259331
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x8)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=8
SLURM_JOBID=3418834
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=32
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e11.hkn0531
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn[0531,0605,0608,0621,0728,0812,0814,0817]
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=32
SLURM_NNODES=8
SLURM_SUBMIT_HOST=hkn1990.localdomain
SLURM_JOB_ID=3418834
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dynamics_maskgit_8_node
SLURM_NTASKS_PER_NODE=4
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0531,0605,0608,0621,0728,0812,0814,0817]
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
2025-08-13 14:03:12.754648: E external/xla/xla/stream_executor/cuda/cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
2025-08-13 14:03:12.754945: E external/xla/xla/stream_executor/cuda/cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
2025-08-13 14:03:12.754961: E external/xla/xla/stream_executor/cuda/cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
2025-08-13 14:03:12.755393: E external/xla/xla/stream_executor/cuda/cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    backend = _init_backend(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 158, in <module>
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    backend = registration.factory()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 158, in <module>
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 158, in <module>
    return xla_client.make_c_api_client(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jaxlib/xla_client.py", line 153, in make_c_api_client
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return _xla.get_c_api_client(plugin_name, options, distributed_client)
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 158, in <module>
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    num_devices = jax.device_count()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 907, in device_count
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    return int(get_backend(backend).device_count())
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 876, in get_backend
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
    return _get_backend_uncached(platform)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 855, in _get_backend_uncached
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    bs = backends()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 758, in backends
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
2025-08-13 14:08:13.373538: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service.cc:1798] Shutdown barrier in coordination service has failed:
DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']
This suggests that the workers are out of sync. Either at least one worker (a) crashed early due to program error or scheduler events (e.g. preemption, eviction), (b) was too fast in its execution, or (c) too slow / hanging. Check the logs (both the program and scheduler events) for an earlier error to identify the root cause.
2025-08-13 14:08:13.373564: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service.cc:1839] Use error polling to propagate the following error to all tasks: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648'] [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.373830: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.373806: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.373972: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Failed to disconnect from coordination service with status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/ShutdownTask [type.googleapis.com/tensorflow.CoordinationServiceError='']Proceeding with agent shutdown anyway. This is usually caused by an earlier error during execution. Check the logs of (a) this task, (b) the leader (usually slice 0 task 0) and (c) the scheduler (e.g. preemption, eviction) for an earlier error to debug further. [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374005: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.373979: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Failed to disconnect from coordination service with status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/ShutdownTask [type.googleapis.com/tensorflow.CoordinationServiceError='']Proceeding with agent shutdown anyway. This is usually caused by an earlier error during execution. Check the logs of (a) this task, (b) the leader (usually slice 0 task 0) and (c) the scheduler (e.g. preemption, eviction) for an earlier error to debug further. [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374003: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374236: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374055: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374175: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374101: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374282: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

2025-08-13 14:08:13.374175: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374096: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374298: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
2025-08-13 14:08:13.374440: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374446: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374299: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374317: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374276: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374215: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374160: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374372: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374510: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374639: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374520: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374599: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374375: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374697: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

2025-08-13 14:08:13.374871: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

2025-08-13 14:08:13.374620: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374414: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374600: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374669: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374823: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

2025-08-13 14:08:13.374758: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374535: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.374784: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374805: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

2025-08-13 14:08:13.374945: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

2025-08-13 14:08:13.374790: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374890: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374801: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374866: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374963: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

2025-08-13 14:08:13.374985: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374968: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

2025-08-13 14:08:13.375000: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374968: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.374969: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.375006: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.375018: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.375020: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::7350480505452186648::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/32.
The first task at the barrier: /job:jax_worker/replica:0/task:2. Some timed out task names:
/job:jax_worker/replica:0/task:29
/job:jax_worker/replica:0/task:17
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::7350480505452186648']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-13 14:08:13.453060: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.453091: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
2025-08-13 14:08:13.453144: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.453164: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:{"created":"@1755086893.452946699","description":"Error received from peer ipv4:10.0.1.99:64210","file":"external/com_github_grpc_grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Socket closed","grpc_status":14}
:{"created":"@1755086893.453054591","description":"Error received from peer ipv4:10.0.1.99:64210","file":"external/com_github_grpc_grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Socket closed","grpc_status":14}
2025-08-13 14:08:13.453150: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.453220: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
2025-08-13 14:08:13.453124: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.453152: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:{"created":"@1755086893.452915579","description":"Error received from peer ipv4:10.0.1.99:64210","file":"external/com_github_grpc_grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Socket closed","grpc_status":14}
:{"created":"@1755086893.453028863","description":"Error received from peer ipv4:10.0.1.99:64210","file":"external/com_github_grpc_grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Socket closed","grpc_status":14}
2025-08-13 14:08:13.453301: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-13 14:08:13.453359: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:{"created":"@1755086893.453059547","description":"Error received from peer ipv4:10.0.1.99:64210","file":"external/com_github_grpc_grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Socket closed","grpc_status":14}
srun: error: hkn0531: tasks 0-3: Aborted (core dumped)
srun: error: hkn0814: tasks 25,27: Aborted (core dumped)
srun: error: hkn0728: tasks 16,18: Aborted (core dumped)
srun: error: hkn0605: tasks 4-5: Aborted (core dumped)
srun: error: hkn0608: tasks 8,10: Aborted (core dumped)
srun: error: hkn0817: tasks 29-30: Aborted (core dumped)
srun: error: hkn0812: tasks 20,22: Aborted (core dumped)
srun: error: hkn0621: tasks 13,15: Aborted (core dumped)
srun: error: hkn0728: task 19: Aborted (core dumped)
srun: error: hkn0814: task 24: Aborted (core dumped)
srun: error: hkn0605: task 7: Aborted (core dumped)
srun: error: hkn0608: task 11: Aborted (core dumped)
srun: error: hkn0817: task 31: Aborted (core dumped)
srun: error: hkn0621: task 14: Aborted (core dumped)
srun: error: hkn0812: task 23: Aborted (core dumped)
srun: error: hkn0814: task 26: Aborted (core dumped)
srun: error: hkn0728: task 17: Aborted (core dumped)
srun: error: hkn0608: task 9: Aborted (core dumped)
srun: error: hkn0605: task 6: Aborted (core dumped)
srun: error: hkn0817: task 28: Aborted (core dumped)
srun: error: hkn0812: task 21: Aborted (core dumped)
srun: error: hkn0621: task 12: Aborted (core dumped)

============================= JOB FEEDBACK =============================

Job ID: 3418834
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 134)
Partition: accelerated
Nodes: 8
Cores per node: 24
Nodelist: hkn[0531,0605,0608,0621,0728,0812,0814,0817]
CPU Utilized: 00:02:03
CPU Efficiency: 0.18% of 19:21:36 core-walltime
Job Wall-clock time: 00:06:03
Starttime: Wed Aug 13 14:02:11 2025
Endtime: Wed Aug 13 14:08:14 2025
Memory Utilized: 11.91 GB (estimated maximum)
Memory Efficiency: 0.00% of 0.00 MB (0.00 MB/node)
Energy Consumed: 1494485 Joule / 415.134722222222 Watthours
Average node power draw: 4117.03856749311 Watt