[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/causal/dynamics-cotraining/train_dynamics_causal_8_node_3415137.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/causal/dynamics-cotraining/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/causal/dynamics-cotraining/%x_%j.log
#SBATCH --job-name=train_dynamics_causal_8_node
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi



# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/holiday/causal/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

# tokenizer with the new structure supporting larger ffn_dim
tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401

env | grep SLURM

srun python train_dynamics.py \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=256 \
    --init_lr=0 \
    --dyna_type=causal \
    --max_lr=8e-5 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=dynamics-causal-8-node-$slurm_job_id \
    --tags dynamics causal 8-node post-launch-main \
    --entity instant-uv \
    --project jafar \
    --dyna_dim=1024 \
    --dyna_num_blocks=16 \
    --dyna_num_heads=16 \
    --dyna_ffn_dim=4096 \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x8)
SLURM_JOB_UID=999226
SLURM_TASK_PID=2857600
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar
SLURMD_NODENAME=hkn0627
SLURM_JOB_START_TIME=1754924127
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1755096927
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x8)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=8
SLURM_JOBID=3415137
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=32
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e30.hkn0627
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn[0627-0634]
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=32
SLURM_NNODES=8
SLURM_SUBMIT_HOST=hkn1993.localdomain
SLURM_JOB_ID=3415137
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dynamics_causal_8_node
SLURM_NTASKS_PER_NODE=4
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0627-0634]
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.19.11
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/wandb/run-20250811_165632-3415137
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run dynamics-causal-8-node-3415137
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/3415137
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 76000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/076000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 77000
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/077000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
WARNING:absl:Missing metrics for step 78000
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 78000
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 78000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 78000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401/078000/metrics/metrics not found.
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
2025-08-11 16:58:28.958932: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-11 16:58:29.297694: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-11 16:58:29.297744: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-11 16:58:29.665801: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-11 16:58:29.922796: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-11 16:58:34.964741: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-11 16:59:33.110313: W external/xla/xla/tsl/framework/bfc_allocator.cc:501] Allocator (GPU_0_bfc) ran out of memory trying to allocate 17.79GiB (rounded to 19101157120)requested by op 
If the cause is memory fragmentation maybe the environment variable 'TF_GPU_ALLOCATOR=cuda_malloc_async' will improve the situation. 
Current allocation summary follows.
Current allocation summary follows.
2025-08-11 16:59:33.115071: W external/xla/xla/tsl/framework/bfc_allocator.cc:512] ****************************************____________________________________________________________
E0811 16:59:33.115839 3967043 pjrt_stream_executor_client.cc:2916] Execution of replica 0 failed: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 19101156952 bytes. [tf-allocator-allocation-error='']
jax.errors.SimplifiedTraceback: For simplicity, JAX has removed its internal frames from the traceback of the following exception. Set JAX_TRACEBACK_FILTERING=off to include these.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/train_dynamics.py", line 365, in <module>
    loss, recon, metrics = train_step(optimizer.model, optimizer, inputs)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/flax/nnx/transforms/compilation.py", line 431, in __call__
    pure_args_out, pure_kwargs_out, pure_out = self.jitted_fn(
jaxlib._jax.XlaRuntimeError: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 19101156952 bytes.
Filtering out episode with length 1, which is shorter than the requested sequence length 16.
Filtering out episode with length 12, which is shorter than the requested sequence length 16.
Filtering out episode with length 4, which is shorter than the requested sequence length 16.
2025-08-11 16:59:33.834118: W external/xla/xla/tsl/framework/bfc_allocator.cc:501] Allocator (GPU_2_bfc) ran out of memory trying to allocate 17.79GiB (rounded to 19101157120)requested by op 
If the cause is memory fragmentation maybe the environment variable 'TF_GPU_ALLOCATOR=cuda_malloc_async' will improve the situation. 
Current allocation summary follows.
Current allocation summary follows.
2025-08-11 16:59:33.838886: W external/xla/xla/tsl/framework/bfc_allocator.cc:512] ****************************************____________________________________________________________
E0811 16:59:33.839561 3592077 pjrt_stream_executor_client.cc:2916] Execution of replica 0 failed: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 19101156952 bytes. [tf-allocator-allocation-error='']
jax.errors.SimplifiedTraceback: For simplicity, JAX has removed its internal frames from the traceback of the following exception. Set JAX_TRACEBACK_FILTERING=off to include these.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/train_dynamics.py", line 365, in <module>
    loss, recon, metrics = train_step(optimizer.model, optimizer, inputs)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/flax/nnx/transforms/compilation.py", line 431, in __call__
    pure_args_out, pure_kwargs_out, pure_out = self.jitted_fn(
jaxlib._jax.XlaRuntimeError: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 19101156952 bytes.
Filtering out episode with length 6, which is shorter than the requested sequence length 16.
2025-08-11 16:59:36.876421: W external/xla/xla/tsl/framework/bfc_allocator.cc:501] Allocator (GPU_3_bfc) ran out of memory trying to allocate 17.79GiB (rounded to 19101157120)requested by op 
If the cause is memory fragmentation maybe the environment variable 'TF_GPU_ALLOCATOR=cuda_malloc_async' will improve the situation. 
Current allocation summary follows.
Current allocation summary follows.
2025-08-11 16:59:36.881327: W external/xla/xla/tsl/framework/bfc_allocator.cc:512] ****************************************____________________________________________________________
E0811 16:59:36.882089 2857674 pjrt_stream_executor_client.cc:2916] Execution of replica 0 failed: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 19101156952 bytes. [tf-allocator-allocation-error='']
jax.errors.SimplifiedTraceback: For simplicity, JAX has removed its internal frames from the traceback of the following exception. Set JAX_TRACEBACK_FILTERING=off to include these.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/train_dynamics.py", line 365, in <module>
    loss, recon, metrics = train_step(optimizer.model, optimizer, inputs)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/flax/nnx/transforms/compilation.py", line 431, in __call__
    pure_args_out, pure_kwargs_out, pure_out = self.jitted_fn(
jaxlib._jax.XlaRuntimeError: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 19101156952 bytes.
Filtering out episode with length 1, which is shorter than the requested sequence length 16.
2025-08-11 17:04:35.391971: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service.cc:1798] Shutdown barrier in coordination service has failed:
DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']
This suggests that the workers are out of sync. Either at least one worker (a) crashed early due to program error or scheduler events (e.g. preemption, eviction), (b) was too fast in its execution, or (c) too slow / hanging. Check the logs (both the program and scheduler events) for an earlier error to identify the root cause.
2025-08-11 17:04:35.392019: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service.cc:1839] Use error polling to propagate the following error to all tasks: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
2025-08-11 17:04:35.392431: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392591: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
2025-08-11 17:04:35.392357: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392364: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392584: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622'] [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392462: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392487: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392568: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392592: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392595: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392237: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392571: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392557: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392523: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392856: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.393082: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
2025-08-11 17:04:35.392824: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.393025: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392789: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392425: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392669: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392703: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392724: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392602: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392843: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392747: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392791: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392779: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392873: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.393015: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
2025-08-11 17:04:35.392881: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392823: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392670: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392710: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392781: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392839: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392788: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392785: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.394553: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.394705: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
2025-08-11 17:04:35.393190: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392684: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392986: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.393317: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392960: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392918: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392807: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.393165: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392856: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392976: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392848: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.393088: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392879: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392923: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.393680: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.392837: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.393262: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.393140: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.393057: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.393823: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

2025-08-11 17:04:35.392965: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.392991: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-08-11 17:04:35.395507: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:429] Polled an error from coordination service (this can be an error from this or another task).
2025-08-11 17:04:35.395656: F external/xla/xla/pjrt/distributed/client.h:88] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::2525437949938900622::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 3/32.
The first task at the barrier: /job:jax_worker/replica:0/task:8. Some timed out task names:
/job:jax_worker/replica:0/task:4
/job:jax_worker/replica:0/task:16
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1dShutdown::2525437949938900622']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 11 leaked shared_memory objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 10 leaked shared_memory objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 10 leaked shared_memory objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
srun: error: hkn0627: tasks 2-3: Aborted (core dumped)
srun: error: hkn0630: task 14: Aborted (core dumped)
srun: error: hkn0628: tasks 5-6: Aborted (core dumped)
srun: error: hkn0632: tasks 21-22: Aborted (core dumped)
srun: error: hkn0629: tasks 8-10: Aborted (core dumped)
srun: error: hkn0631: tasks 17-19: Aborted (core dumped)
srun: error: hkn0633: tasks 25-27: Aborted (core dumped)
srun: error: hkn0629: task 11: Aborted (core dumped)
srun: error: hkn0627: tasks 0-1: Aborted (core dumped)
srun: error: hkn0633: task 24: Aborted (core dumped)
srun: error: hkn0631: task 16: Aborted (core dumped)
srun: error: hkn0632: tasks 20,23: Aborted (core dumped)
srun: error: hkn0628: tasks 4,7: Aborted (core dumped)
srun: error: hkn0630: tasks 12-13,15: Aborted (core dumped)
srun: error: hkn0634: tasks 28-31: Aborted (core dumped)

============================= JOB FEEDBACK =============================

Job ID: 3415137
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 134)
Partition: accelerated
Nodes: 8
Cores per node: 24
Nodelist: hkn[0627-0634]
CPU Utilized: 03:46:33
CPU Efficiency: 12.87% of 1-05:20:00 core-walltime
Job Wall-clock time: 00:09:10
Starttime: Mon Aug 11 16:55:27 2025
Endtime: Mon Aug 11 17:04:37 2025
Memory Utilized: 237.17 GB (estimated maximum)
Memory Efficiency: 0.00% of 0.00 MB (0.00 MB/node)
Energy Consumed: 2586926 Joule / 718.590555555556 Watthours
Average node power draw: 4703.50181818182 Watt