[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/breakout/dyn/train_dyn_default_gt_actions_breakout_longer_3519481.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=06:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/breakout/dyn/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/breakout/dyn/%x_%j.log
#SBATCH --job-name=train_dyn_default_gt_actions_breakout_longer
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
# source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_breakout/breakout_episodes_perfect_big/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_breakout/breakout_episodes_perfect_big/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

tokenizer_checkpoint=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/tokenizer/interactive/3518963

env | grep SLURM

srun python jasmine/train_dynamics.py \
    --save_ckpt \
    --image_height=10 \
    --image_width=10 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=120 \
    --init_lr=0 \
    --max_lr=3e-5 \
    --log_image_interval=250 \
    --log_checkpoint_interval=250 \
    --dyna_type=maskgit \
    --log \
    --name=breakout-dyn-50ksteps-gt-actions-lr-3e-5-$slurm_job_id \
    --tags dyn breakout 50ksteps 3e-5 \
    --entity instant-uv \
    --project jafar \
    --patch_size 4 \
    --lam_patch_size 4 \
    --warmup_steps 100 \
    --wsd_decay_steps 1000 \
    --num_steps 5000 \
    --use_gt_actions \
    --data_dir $array_records_dir_train \
    --val_data_dir $array_records_dir_val \
    --tokenizer_checkpoint $tokenizer_checkpoint \
    --val_interval 750 \
    --eval_full_frame \
SLURM_STEP_NUM_TASKS=1
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=1
SLURM_JOB_UID=999226
SLURM_TASK_PID=3542946
SLURM_JOB_GPUS=2
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs_2
SLURMD_NODENAME=hkn0503
SLURM_JOB_START_TIME=1758723218
SLURM_STEP_NODELIST=hkn0603
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1758744818
SLURM_PMI2_SRUN_PORT=44785
SLURM_CPUS_ON_NODE=6
SLURM_JOB_CPUS_PER_NODE=6
SLURM_GPUS_ON_NODE=1
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=1
SLURM_STEPID=4294967290
SLURM_JOBID=3519481
SLURM_PTY_PORT=40815
SLURM_JOB_QOS=normal
SLURM_LAUNCH_NODE_IPADDR=10.0.7.198
SLURM_PTY_WIN_ROW=52
SLURM_PMI2_PROC_MAPPING=(vector,(0,1,1))
SLURMD_DEBUG=2
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=1
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e10.hkn0503
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SRUN_COMM_HOST=10.0.7.198
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_PTY_WIN_COL=131
SLURM_NODELIST=hkn0503
SLURM_SRUN_COMM_PORT=45511
SLURM_STEP_ID=4294967290
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=1
SLURM_NNODES=1
SLURM_SUBMIT_HOST=hkn0603.localdomain
SLURM_JOB_ID=3519481
SLURM_NODEID=0
SLURM_STEP_NUM_NODES=1
SLURM_STEP_TASKS_PER_NODE=1
SLURM_MPI_TYPE=pmi2
SLURM_PMI2_STEP_NODES=hkn0603
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dyn_default_gt_actions_breakout_longer
SLURM_NTASKS_PER_NODE=1
SLURM_STEP_LAUNCHER_PORT=45511
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn0503
GpuFreq=control_disabled
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: creating run
wandb: Tracking run with wandb version 0.22.0
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs_2/wandb/run-20250924_161414-2iqzypsg
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run breakout-dyn-50ksteps-gt-actions-lr-3e-5-3519481
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/2iqzypsg
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1269: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
Running on 1 devices.
Counting all components: ['action_embed', 'dynamics', 'tokenizer']
Parameter counts:
{'action_embed': 192, 'dynamics': 26555904, 'tokenizer': 33750256, 'total': 60306352}
WARNING:absl:[process=0][thread=MainThread][operation_id=1] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=1] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=2] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=2] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
Total memory size: 1.7 GB, Output size: 0.7 GB, Temp size: 1.0 GB, Argument size: 0.7 GB, Host temp size: 0.0 GB.
FLOPs: 3.479e+10, Bytes: 3.039e+10 (28.3 GB), Intensity: 1.1 FLOPs/byte
Starting training from step 0...

Memstats: After params initialized:
	Using (GB) 0.84 / 38.7 (2.170543%) on cuda:0
Saved checkpoint at step 250
Saved checkpoint at step 500
Calculating validation metrics...
WARNING:absl:[process=0][thread=MainThread][operation_id=3] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=3] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=4] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=4] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=5] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=5] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=6] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=6] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/000750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=7] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=7] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=8] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=8] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=9] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=9] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=10] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=10] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/001750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=11] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=11] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=12] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=12] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=13] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=13] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=14] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=14] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/002750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=15] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=15] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=16] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=16] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=17] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=17] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=18] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=18] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/003750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=19] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=19] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004250) to end with ".orbax-checkpoint-tmp".
WARNING:absl:[process=0][thread=MainThread][operation_id=20] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:[process=0][thread=MainThread][operation_id=20] _SignalingThread.join() waiting for signals ([<HandlerAwaitableSignal.STEP_DIRECTORY_CREATION: 'step_directory_creation'>, <HandlerAwaitableSignal.ITEM_DIRECTORY_CREATION: 'item_directory_creation'>]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004750 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004750) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/005000 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/005000) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004500 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004500) to end with ".orbax-checkpoint-tmp".
WARNING:absl:Path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004250 could not be identified as a temporary checkpoint path using <class 'orbax.checkpoint._src.path.atomicity.AtomicRenameTemporaryPath'>. Got error: Expected AtomicRenameTemporaryPath (/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/train_dyn_default_gt_actions_breakout_longer/3519481/004250) to end with ".orbax-checkpoint-tmp".
Step 750, validation loss: 0.03621530532836914
Saved checkpoint at step 750
Saved checkpoint at step 1000
Saved checkpoint at step 1250
Calculating validation metrics...
Step 1500, validation loss: 0.0024774176999926567
Saved checkpoint at step 1500
Saved checkpoint at step 1750
Saved checkpoint at step 2000
Calculating validation metrics...
Step 2250, validation loss: 0.003023855620995164
Saved checkpoint at step 2250
Saved checkpoint at step 2500
Saved checkpoint at step 2750
Calculating validation metrics...
Step 3000, validation loss: 0.0015773066552355886
Saved checkpoint at step 3000
Saved checkpoint at step 3250
Saved checkpoint at step 3500
Calculating validation metrics...
Step 3750, validation loss: 0.0009347246959805489
Saved checkpoint at step 3750
Saved checkpoint at step 4000
Saved checkpoint at step 4250
Calculating validation metrics...
Step 4500, validation loss: 0.00037653770414181054
Saved checkpoint at step 4500
Saved checkpoint at step 4750
Saved checkpoint at step 5000
wandb: 
wandb: 🚀 View run breakout-dyn-50ksteps-gt-actions-lr-3e-5-3519481 at: https://wandb.ai/instant-uv/jafar/runs/2iqzypsg
wandb: Find logs at: ../../../../../hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs_2/wandb/run-20250924_161414-2iqzypsg/logs
W0924 16:28:42.152223 3543074 pjrt_client.cc:1469] WatchJobStateAsync failed for task goo.gle/debugstr  job_name: "jax_worker": UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:10.0.1.71:62457: Failed to connect to remote host: Connection refused
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/WatchJobState:
:UNKNOWN:Error received from peer  {grpc_status:14, grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:10.0.1.71:62457: Failed to connect to remote host: Connection refused"}

============================= JOB FEEDBACK =============================

Job ID: 3519481
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: COMPLETED (exit code 0)
Partition: accelerated
Nodes: 1
Cores per node: 6
Nodelist: hkn0503
CPU Utilized: 00:17:02
CPU Efficiency: 18.82% of 01:30:30 core-walltime
Job Wall-clock time: 00:15:05
Starttime: Wed Sep 24 16:13:38 2025
Endtime: Wed Sep 24 16:28:43 2025
Memory Utilized: 7.18 GB
Memory Efficiency: 5.91% of 121.46 GB (121.46 GB/node)
Energy Consumed: 1172358 Joule / 325.655 Watthours
Average node power draw: 1295.42320441989 Watt