[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/causal/dynamics-cotraining/train_dynamics_causal_8_node_3412349.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/causal/dynamics-cotraining/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/causal/dynamics-cotraining/%x_%j.log
#SBATCH --job-name=train_dynamics_causal_8_node
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi



# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/holiday/causal/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

# tokenizer with the new structure supporting larger ffn_dim
tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607

env | grep SLURM

srun python train_dynamics.py \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=320 \
    --init_lr=0 \
    --dyna_type=causal \
    --max_lr=8e-5 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=dynamics-causal-8-node-$slurm_job_id \
    --tags dynamics causal 8-node post-launch-main \
    --entity instant-uv \
    --project jafar \
    --dyna_dim=1024 \
    --dyna_num_blocks=16 \
    --dyna_num_heads=16 \
    --dyna_ffn_dim=4096 \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
/var/spool/slurmd/job3412349/slurm_script: line 42: .venv/bin/activate: No such file or directory
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x8)
SLURM_JOB_UID=999226
SLURM_TASK_PID=1599003
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs
SLURMD_NODENAME=hkn0405
SLURM_JOB_START_TIME=1754831725
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1755004525
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x8)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=8
SLURM_JOBID=3412349
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=32
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e12.hkn0405
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn[0405-0407,0410,0413,0415,0417-0418]
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=32
SLURM_NNODES=8
SLURM_SUBMIT_HOST=hkn1990.localdomain
SLURM_JOB_ID=3412349
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dynamics_causal_8_node
SLURM_NTASKS_PER_NODE=4
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0405-0407,0410,0413,0415,0417-0418]
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: creating run
wandb: Tracking run with wandb version 0.19.11
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/wandb/run-20250810_151635-3412349
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run dynamics-causal-8-node-3412349
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/3412349
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Dropping 18 examples of 89394 examples (shard 32).
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 140000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 139000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 140000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/140000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 139000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/139000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 141000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/141000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3404607/020000/metrics/metrics not found.
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
Running on 32 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 271932416, 'lam': 35115232, 'tokenizer': 33750256, 'total': 340797904}
Starting training from step 0...
2025-08-10 15:18:43.612156: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:18:43.666530: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:18:44.322704: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:18:44.817283: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:18:44.817329: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:18:49.742321: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-08-10 15:18:56.404772: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_1_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.404765: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_2_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.412973: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_0_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.412986: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_1_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.412955: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_2_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.413034: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_3_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.414405: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_1_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.414341: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_0_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.414347: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_1_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.414344: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_2_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.414349: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_3_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.415071: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_0_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.415022: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_2_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.415154: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_3_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.416958: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_3_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.417132: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_0_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.417043: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_1_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.417039: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_2_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.418270: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_0_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.418266: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_1_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.418271: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_2_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.418274: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_3_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.441566: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_0_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.441568: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_2_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.441583: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_3_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.446259: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_0_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.446257: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_2_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.446259: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_3_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.446390: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_1_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.451359: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_1_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.527548: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_0_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2025-08-10 15:18:56.531572: W external/xla/xla/tsl/framework/bfc_allocator.cc:310] Allocator (GPU_3_bfc) ran out of memory trying to allocate 21.27GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.