[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/causal/dynamics-cotraining/train_dynamics_causal_8_node_3412356.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/causal/dynamics-cotraining/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/causal/dynamics-cotraining/%x_%j.log
#SBATCH --job-name=train_dynamics_causal_8_node
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi



# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/holiday/causal/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

# tokenizer with the new structure supporting larger ffn_dim
tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607

env | grep SLURM

srun python train_dynamics.py \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=256 \
    --init_lr=0 \
    --dyna_type=causal \
    --max_lr=8e-5 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=dynamics-causal-8-node-$slurm_job_id \
    --tags dynamics causal 8-node post-launch-main \
    --entity instant-uv \
    --project jafar \
    --dyna_dim=1024 \
    --dyna_num_blocks=16 \
    --dyna_num_heads=16 \
    --dyna_ffn_dim=4096 \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
/var/spool/slurmd/job3412356/slurm_script: line 42: .venv/bin/activate: No such file or directory
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x8)
SLURM_JOB_UID=999226
SLURM_TASK_PID=980352
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs
SLURMD_NODENAME=hkn0404
SLURM_JOB_START_TIME=1754832109
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1755004909
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x8)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=8
SLURM_JOBID=3412356
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=32
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e11.hkn0404
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn[0404,0521-0527]
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=32
SLURM_NNODES=8
SLURM_SUBMIT_HOST=hkn1990.localdomain
SLURM_JOB_ID=3412356
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dynamics_causal_8_node
SLURM_NTASKS_PER_NODE=4
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0404,0521-0527]
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.19.11
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/wandb/run-20250810_152243-3412356
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run dynamics-causal-8-node-3412356
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/3412356
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
2025-08-10 15:24:42.551849: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:24:43.942931: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:24:44.153932: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:24:44.153977: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:24:48.070016: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:24:50.048268: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:25:50.683723: W external/xla/xla/tsl/framework/bfc_allocator.cc:501] Allocator (GPU_1_bfc) ran out of memory trying to allocate 16.04GiB (rounded to 17224249088)requested by op 
If the cause is memory fragmentation maybe the environment variable 'TF_GPU_ALLOCATOR=cuda_malloc_async' will improve the situation. 
Current allocation summary follows.
Current allocation summary follows.
2025-08-10 15:25:50.688554: W external/xla/xla/tsl/framework/bfc_allocator.cc:512] **************************************************__________________________________________________
E0810 15:25:50.689309 1675360 pjrt_stream_executor_client.cc:2916] Execution of replica 0 failed: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 17224248880 bytes. [tf-allocator-allocation-error='']
jax.errors.SimplifiedTraceback: For simplicity, JAX has removed its internal frames from the traceback of the following exception. Set JAX_TRACEBACK_FILTERING=off to include these.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 365, in <module>
    loss, recon, metrics = train_step(optimizer.model, optimizer, inputs)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/flax/nnx/transforms/compilation.py", line 431, in __call__
    pure_args_out, pure_kwargs_out, pure_out = self.jitted_fn(
jaxlib._jax.XlaRuntimeError: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 17224248880 bytes.