[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/train_tokenizer_batch_size_scaling_8_node_3528969.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_8_node
#SBATCH --mem=100G
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_minecraft/open_ai_minecraft_arrayrecords_chunked_train_val_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_minecraft/open_ai_minecraft_arrayrecords_chunked_train_val_split/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python jasmine/train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=384 \
    --max_lr=3e-4 \
    --log_image_interval=1000 \
    --log_checkpoint_interval=1000 \
    --log \
    --patch_size=16 \
    --name=minecraft-tokenizer-default-$slurm_job_id \
    --tags minecraft-tokenizer default \
    --entity instant-uv \
    --project jafar \
    --data_dir $array_records_dir_train \
    --val_data_dir $array_records_dir_val &

child_pid=$!

wait $child_pid
/var/spool/slurmd/job3528969/slurm_script: line 41: .venv/bin/activate: No such file or directory
SLURM_STEP_NUM_TASKS=1
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x8)
SLURM_JOB_UID=999226
SLURM_TASK_PID=424804
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs
SLURMD_NODENAME=hkn0627
SLURM_JOB_START_TIME=1759190741
SLURM_STEP_NODELIST=hkn0402
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1759363541
SLURM_PMI2_SRUN_PORT=35693
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x8)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=8
SLURM_STEPID=4294967290
SLURM_JOBID=3528969
SLURM_PTY_PORT=38767
SLURM_LAUNCH_NODE_IPADDR=10.0.7.199
SLURM_JOB_QOS=normal
SLURM_PTY_WIN_ROW=47
SLURM_PMI2_PROC_MAPPING=(vector,(0,1,1))
SLURMD_DEBUG=2
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=32
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e30.hkn0627
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SRUN_COMM_HOST=10.0.7.199
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_MEM_PER_NODE=102400
SLURM_PTY_WIN_COL=230
SLURM_NODELIST=hkn[0627-0634]
SLURM_SRUN_COMM_PORT=39059
SLURM_STEP_ID=4294967290
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=32
SLURM_NNODES=8
SLURM_SUBMIT_HOST=hkn0402.localdomain
SLURM_JOB_ID=3528969
SLURM_NODEID=0
SLURM_STEP_NUM_NODES=1
SLURM_STEP_TASKS_PER_NODE=1
SLURM_MPI_TYPE=pmi2
SLURM_PMI2_STEP_NODES=hkn0402
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_tokenizer_batch_size_scaling_8_node
SLURM_NTASKS_PER_NODE=4
SLURM_STEP_LAUNCHER_PORT=39059
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0627-0634]
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
E0930 02:06:32.154106  420622 cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
E0930 02:06:32.154433  420621 cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 812, in backends
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 812, in backends
E0930 02:06:32.155359  420620 cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
E0930 02:06:32.155375  420623 cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 812, in backends
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 812, in backends
    backend = _init_backend(platform)
    backend = _init_backend(platform)
    backend = _init_backend(platform)
              ^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 896, in _init_backend
              ^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 896, in _init_backend
              ^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 896, in _init_backend
    backend = _init_backend(platform)
    backend = registration.factory()
              ^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 896, in _init_backend
              ^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 559, in make_pjrt_c_api_client
    backend = registration.factory()
    backend = registration.factory()
    backend = registration.factory()
              ^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 559, in make_pjrt_c_api_client
              ^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 559, in make_pjrt_c_api_client
              ^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 559, in make_pjrt_c_api_client
    return xla_client.make_c_api_client(
    return xla_client.make_c_api_client(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jaxlib/xla_client.py", line 156, in make_c_api_client
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jaxlib/xla_client.py", line 156, in make_c_api_client
    return _xla.get_c_api_client(
    return xla_client.make_c_api_client(
           ^^^^^^^^^^^^^^^^^^^^^^
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 574, in <module>
    return _xla.get_c_api_client(
           ^^^^^^^^^^^^^^^^^^^^^^
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

    return xla_client.make_c_api_client(
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 574, in <module>
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jaxlib/xla_client.py", line 156, in make_c_api_client
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jaxlib/xla_client.py", line 156, in make_c_api_client
    main(args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 273, in main
    num_devices = jax.device_count()
    main(args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 273, in main
    num_devices = jax.device_count()
                  ^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 975, in device_count
    return _xla.get_c_api_client(
                  ^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 975, in device_count
    return _xla.get_c_api_client(
           ^^^^^^^^^^^^^^^^^^^^^^
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 574, in <module>
    return int(get_backend(backend).device_count())
               ^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 944, in get_backend
           ^^^^^^^^^^^^^^^^^^^^^^
jaxlib._jax.XlaRuntimeError: FAILED_PRECONDITION: No visible GPU devices.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 574, in <module>
    return int(get_backend(backend).device_count())
               ^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 944, in get_backend
    return _get_backend_uncached(platform)
    main(args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 273, in main
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 923, in _get_backend_uncached
    main(args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 273, in main
    num_devices = jax.device_count()
    return _get_backend_uncached(platform)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 923, in _get_backend_uncached
    num_devices = jax.device_count()
    bs = backends()
         ^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 828, in backends
                  ^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 975, in device_count
                  ^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 975, in device_count
    bs = backends()
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
         ^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 828, in backends
    return int(get_backend(backend).device_count())
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
    return int(get_backend(backend).device_count())
               ^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 944, in get_backend
               ^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 944, in get_backend
    return _get_backend_uncached(platform)
    return _get_backend_uncached(platform)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 923, in _get_backend_uncached
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 923, in _get_backend_uncached
    bs = backends()
         ^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 828, in backends
    bs = backends()
         ^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 828, in backends
    raise RuntimeError(err_msg)
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
RuntimeError: Unable to initialize backend 'cuda': FAILED_PRECONDITION: No visible GPU devices. (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
W0930 02:06:32.782466 3067532 platform_util.cc:218] unable to create StreamExecutor for CUDA:3: CUDA error: Failed call to cuDeviceGet: CUDA_ERROR_INVALID_DEVICE: invalid device ordinal
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 812, in backends
    backend = _init_backend(platform)
              ^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 896, in _init_backend
    backend = registration.factory()
              ^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 559, in make_pjrt_c_api_client
    return xla_client.make_c_api_client(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jaxlib/xla_client.py", line 156, in make_c_api_client
    return _xla.get_c_api_client(
           ^^^^^^^^^^^^^^^^^^^^^^
jaxlib._jax.XlaRuntimeError: INTERNAL: no supported devices found for platform CUDA

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 574, in <module>
    main(args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 273, in main
    num_devices = jax.device_count()
                  ^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 975, in device_count
    return int(get_backend(backend).device_count())
               ^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 944, in get_backend
    return _get_backend_uncached(platform)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 923, in _get_backend_uncached
    bs = backends()
         ^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 828, in backends
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': INTERNAL: no supported devices found for platform CUDA (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 812, in backends
    backend = _init_backend(platform)
              ^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 896, in _init_backend
    backend = registration.factory()
              ^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 559, in make_pjrt_c_api_client
    return xla_client.make_c_api_client(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jaxlib/xla_client.py", line 156, in make_c_api_client
    return _xla.get_c_api_client(
           ^^^^^^^^^^^^^^^^^^^^^^
jaxlib._jax.XlaRuntimeError: INTERNAL: Getting local topologies failed: Error 1: GetKeyValue() timed out with key: cuda:local_topology/cuda/4 and duration: 2m

Error 2: GetKeyValue() timed out with key: cuda:local_topology/cuda/5 and duration: 2m

Error 3: GetKeyValue() timed out with key: cuda:local_topology/cuda/6 and duration: 2m

Error 4: GetKeyValue() timed out with key: cuda:local_topology/cuda/7 and duration: 2m

Error 5: GetKeyValue() timed out with key: cuda:local_topology/cuda/23 and duration: 2m

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 574, in <module>
    main(args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_tokenizer.py", line 273, in main
    num_devices = jax.device_count()
                  ^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 975, in device_count
    return int(get_backend(backend).device_count())
               ^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 944, in get_backend
    return _get_backend_uncached(platform)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 923, in _get_backend_uncached
    bs = backends()
         ^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/jax/_src/xla_bridge.py", line 828, in backends
    raise RuntimeError(err_msg)
RuntimeError: Unable to initialize backend 'cuda': INTERNAL: Getting local topologies failed: Error 1: GetKeyValue() timed out with key: cuda:local_topology/cuda/4 and duration: 2m

Error 2: GetKeyValue() timed out with key: cuda:local_topology/cuda/5 and duration: 2m

Error 3: GetKeyValue() timed out with key: cuda:local_topology/cuda/6 and duration: 2m

Error 4: GetKeyValue() timed out with key: cuda:local_topology/cuda/7 and duration: 2m

Error 5: GetKeyValue() timed out with key: cuda:local_topology/cuda/23 and duration: 2m (you may need to uninstall the failing plugin package, or set JAX_PLATFORMS=cpu to skip this backend.)
E0930 02:11:32.547164  424973 coordination_service.cc:1852] Shutdown barrier in coordination service has failed:
DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']
This suggests that the workers are out of sync. Either at least one worker (a) crashed early due to program error or scheduler events (e.g. preemption, eviction), (b) was too fast in its execution, or (c) too slow / hanging. Check the logs (both the program and scheduler events) for an earlier error to identify the root cause.
E0930 02:11:32.547194  424973 coordination_service.cc:1893] Use error polling to propagate the following error to all tasks: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.547424  425050 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E0930 02:11:32.547626  425039 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.547701  425050 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F0930 02:11:32.547846  425039 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

E0930 02:11:32.547669  420758 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E0930 02:11:32.547873 3815912 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.547825  425036 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E0930 02:11:32.548021 3815911 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.548005  425036 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.548003 1214025 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.548146 3815912 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

E0930 02:11:32.547893  420760 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E0930 02:11:32.547993  420759 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.548061  420758 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.548078  420761 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F0930 02:11:32.548186 3815911 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

E0930 02:11:32.548141 3053933 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.548194 3815908 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.548195  420761 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.548361  357724 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E0930 02:11:32.548325 1214016 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E0930 02:11:32.548375 1224629 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.548412 3053933 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

E0930 02:11:32.548348  357723 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.548300 1214025 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

F0930 02:11:32.548444 3815908 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

E0930 02:11:32.548424 1224631 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.548476  357721 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.548368 1214015 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E0930 02:11:32.548373 3067420 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E0930 02:11:32.548361 3067421 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.548611  420760 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F0930 02:11:32.548525  420759 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.548679 3053932 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.548702  357724 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

F0930 02:11:32.548786 1214016 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

E0930 02:11:32.548570 1224630 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F0930 02:11:32.548676 1224629 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

F0930 02:11:32.548715  357723 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

F0930 02:11:32.548768 1214015 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F0930 02:11:32.549012 1224631 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F0930 02:11:32.549131 1224630 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

F0930 02:11:32.549066  357721 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

F0930 02:11:32.549303 3053932 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F0930 02:11:32.549172 3067420 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F0930 02:11:32.549081 3067421 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F0930 02:11:32.549177 3067347 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Failed to disconnect from coordination service with status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::12456228122194650069::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 6/32.
The first task at the barrier: /job:jax_worker/replica:0/task:5. Some timed out task names:
/job:jax_worker/replica:0/task:14
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::12456228122194650069'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/ShutdownTask [type.googleapis.com/tensorflow.CoordinationServiceError='']Proceeding with agent shutdown anyway. This is usually caused by an earlier error during execution. Check the logs of (a) this task, (b) the leader (usually slice 0 task 0) and (c) the scheduler (e.g. preemption, eviction) for an earlier error to debug further. [type.googleapis.com/tensorflow.CoordinationServiceError='']
W0930 02:11:32.683398  425022 preemption_sync_manager.cc:149] Failed to retrieve preemption notice from coordination service: UNAVAILABLE: Socket closed
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/GetKeyValue:
:UNKNOWN:Error received from peer  {grpc_status:14, grpc_message:"Socket closed"}. This is only expected if one of the tasks is unhealthy. Check the logs for the actual root cause.
E0930 02:11:32.683485  425022 preemption_sync_manager.cc:161] Failed to cancel preemption barrier: FAILED_PRECONDITION: Tried to cancel non-existent barrier PREEMPTION_SYNC_BARRIER. [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.683519  425022 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
W0930 02:11:32.683375 3053927 preemption_sync_manager.cc:149] Failed to retrieve preemption notice from coordination service: UNAVAILABLE: Socket closed
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/GetKeyValue:
:UNKNOWN:Error received from peer  {grpc_message:"Socket closed", grpc_status:14}. This is only expected if one of the tasks is unhealthy. Check the logs for the actual root cause.
E0930 02:11:32.683346  357722 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.683385  357722 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:UNKNOWN:Error received from peer  {grpc_status:14, grpc_message:"Socket closed"}
W0930 02:11:32.683279 1214020 preemption_sync_manager.cc:149] Failed to retrieve preemption notice from coordination service: UNAVAILABLE: Socket closed
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/GetKeyValue:
:UNKNOWN:Error received from peer  {grpc_status:14, grpc_message:"Socket closed"}. This is only expected if one of the tasks is unhealthy. Check the logs for the actual root cause.
E0930 02:11:32.683365 1214020 preemption_sync_manager.cc:161] Failed to cancel preemption barrier: FAILED_PRECONDITION: Tried to cancel non-existent barrier PREEMPTION_SYNC_BARRIER. [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.683403 1214020 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
W0930 02:11:32.683326 3815906 preemption_sync_manager.cc:149] Failed to retrieve preemption notice from coordination service: UNAVAILABLE: Socket closed
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/GetKeyValue:
:UNKNOWN:Error received from peer  {grpc_status:14, grpc_message:"Socket closed"}. This is only expected if one of the tasks is unhealthy. Check the logs for the actual root cause.
F0930 02:11:32.683524  425022 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:UNKNOWN:Error received from peer  {grpc_message:"Socket closed", grpc_status:14}
E0930 02:11:32.683423 3053928 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.683471 3053928 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:UNKNOWN:Error received from peer  {grpc_message:"Socket closed", grpc_status:14}
E0930 02:11:32.683411 3815906 preemption_sync_manager.cc:161] Failed to cancel preemption barrier: FAILED_PRECONDITION: Tried to cancel non-existent barrier PREEMPTION_SYNC_BARRIER. [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.683446 3815906 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.683450 3815906 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:UNKNOWN:Error received from peer  {grpc_message:"Socket closed", grpc_status:14}
E0930 02:11:32.683377 1224632 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.683432 1224632 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:UNKNOWN:Error received from peer  {grpc_status:14, grpc_message:"Socket closed"}
F0930 02:11:32.683407 1214020 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:UNKNOWN:Error received from peer  {grpc_message:"Socket closed", grpc_status:14}
E0930 02:11:32.683461 3053927 preemption_sync_manager.cc:161] Failed to cancel preemption barrier: FAILED_PRECONDITION: Tried to cancel non-existent barrier PREEMPTION_SYNC_BARRIER. [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.683497 3053927 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.683502 3053927 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:UNKNOWN:Error received from peer  {grpc_message:"Socket closed", grpc_status:14}
W0930 02:11:32.683506 3067422 preemption_sync_manager.cc:149] Failed to retrieve preemption notice from coordination service: UNAVAILABLE: Socket closed
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/GetKeyValue:
:UNKNOWN:Error received from peer  {grpc_status:14, grpc_message:"Socket closed"}. This is only expected if one of the tasks is unhealthy. Check the logs for the actual root cause.
E0930 02:11:32.683577 3067422 preemption_sync_manager.cc:161] Failed to cancel preemption barrier: FAILED_PRECONDITION: Tried to cancel non-existent barrier PREEMPTION_SYNC_BARRIER. [type.googleapis.com/tensorflow.CoordinationServiceError='']
E0930 02:11:32.683604 3067422 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F0930 02:11:32.683608 3067422 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: UNAVAILABLE: Failed to send RPC to coordination service. Either the leader task was preempted/died/restarted unexpectedly or this task is experiencing network issues. Check earlier logs from 1) this task, 2) the leader (usually slice 0 task 0), and 3) cluster scheduler to debug further.
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/PollForError:
:UNKNOWN:Error received from peer  {grpc_status:14, grpc_message:"Socket closed"}
srun: error: hkn0628: tasks 4-7: Exited with exit code 1
srun: error: hkn0632: task 23: Exited with exit code 1
srun: error: hkn0632: tasks 20,22: Exited with exit code 1
srun: error: hkn0629: task 11: Exited with exit code 1
srun: error: hkn0633: tasks 25-26: Exited with exit code 1
srun: error: hkn0631: tasks 16,19: Exited with exit code 1
srun: error: hkn0627: tasks 2-3: Exited with exit code 1
srun: error: hkn0630: tasks 12-13: Exited with exit code 1
srun: error: hkn0634: tasks 30-31: Exited with exit code 1
srun: error: hkn0629: tasks 8,10: Exited with exit code 1
srun: error: hkn0632: task 21: Exited with exit code 1
srun: error: hkn0633: task 24: Exited with exit code 1
srun: error: hkn0627: task 1: Exited with exit code 1
srun: error: hkn0631: task 17: Exited with exit code 1
srun: error: hkn0630: task 15: Exited with exit code 1
srun: error: hkn0634: task 29: Exited with exit code 1
srun: error: hkn0627: task 0: Exited with exit code 1
srun: error: hkn0633: task 27: Exited with exit code 1
srun: error: hkn0631: task 18: Exited with exit code 1
srun: error: hkn0629: task 9: Exited with exit code 1
srun: error: hkn0634: task 28: Exited with exit code 1
srun: error: hkn0630: task 14: Exited with exit code 1

============================= JOB FEEDBACK =============================

Job ID: 3528969
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 1)
Partition: accelerated
Nodes: 8
Cores per node: 24
Nodelist: hkn[0627-0634]
CPU Utilized: 00:02:35
CPU Efficiency: 0.23% of 18:46:24 core-walltime
Job Wall-clock time: 00:05:52
Starttime: Tue Sep 30 02:05:41 2025
Endtime: Tue Sep 30 02:11:33 2025
Memory Utilized: 9.99 GB
Memory Efficiency: 1.25% of 800.00 GB (100.00 GB/node)
The task which had the largest memory consumption differs by 110.14% from the average task max memory consumption
Energy Consumed: 1342906 Joule / 373.029444444444 Watthours
Average node power draw: 3815.07386363636 Watt