[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/breakout/dyn/train_dyn_default_breakout_longer_3520775.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/breakout/dyn/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/breakout/dyn/%x_%j.log
#SBATCH --job-name=train_dyn_default_breakout_longer
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
# source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_breakout/breakout_episodes_perfect_big/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_breakout/breakout_episodes_perfect_big/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/dyn-gt-actions/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

tokenizer_checkpoint=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/breakout/tokenizer/interactive/3518963

env | grep SLURM

export PYTHONUNBUFFERED=1

srun python jasmine/train_dynamics.py \
    --save_ckpt \
    --image_height=10 \
    --image_width=10 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=120 \
    --init_lr=0 \
    --max_lr=3e-5 \
    --log_image_interval=1000 \
    --log_checkpoint_interval=1000 \
    --dyna_type=maskgit \
    --log \
    --name=breakout-dyn-default-noise-lvl-gt-actions-$slurm_job_id \
    --tags dyn breakout default noise-lvl \
    --entity instant-uv \
    --project jafar \
    --patch_size 4 \
    --lam_patch_size 4 \
    --warmup_steps 1000 \
    --wsd_decay_steps 10000 \
    --num_steps 5000 \
    --use_gt_actions \
    --data_dir $array_records_dir_train \
    --val_data_dir $array_records_dir_val \
    --tokenizer_checkpoint $tokenizer_checkpoint \
    --val_interval 1000 \
    --eval_full_frame \
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=1
SLURM_JOB_UID=999226
SLURM_TASK_PID=1066133
SLURM_JOB_GPUS=3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs
SLURMD_NODENAME=hkn0406
SLURM_JOB_START_TIME=1758808724
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1758981524
SLURM_CPUS_ON_NODE=6
SLURM_JOB_CPUS_PER_NODE=6
SLURM_GPUS_ON_NODE=1
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=1
SLURM_JOBID=3520775
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=1
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e12.hkn0406
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn0406
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=1
SLURM_NNODES=1
SLURM_SUBMIT_HOST=hkn1991.localdomain
SLURM_JOB_ID=3520775
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dyn_default_breakout_longer
SLURM_NTASKS_PER_NODE=1
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn0406
GpuFreq=control_disabled
Running on 1 devices.
Counting all components: ['action_embed', 'dynamics', 'tokenizer']
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: creating run
wandb: Tracking run with wandb version 0.22.0
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/wandb/run-20250925_155922-5womay8p
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run breakout-dyn-default-noise-lvl-gt-actions-3520775
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/5womay8p
Parameter counts:
{'action_embed': 192, 'dynamics': 26561024, 'tokenizer': 33750256, 'total': 60311472}
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_dynamics.py", line 789, in <module>
    main(args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_dynamics.py", line 423, in main
    optimizer = build_optimizer(genie, args)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/train_dynamics.py", line 161, in build_optimizer
    lr_schedule = get_lr_schedule(
                  ^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/jasmine/utils/train_utils.py", line 28, in get_lr_schedule
    assert (
AssertionError: Warmup and decay period is longer than total steps.
wandb: 
wandb: 🚀 View run breakout-dyn-default-noise-lvl-gt-actions-3520775 at: https://wandb.ai/instant-uv/jafar/runs/5womay8p
wandb: Find logs at: ../../../../../hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/wandb/run-20250925_155922-5womay8p/logs
W0925 15:59:24.633578 1066493 pjrt_client.cc:1469] WatchJobStateAsync failed for task goo.gle/debugstr  job_name: "jax_worker": CANCELLED: CANCELLED
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/WatchJobState:
:UNKNOWN:Error received from peer  {grpc_message:"CANCELLED", grpc_status:1} [type.googleapis.com/tensorflow.DerivedStatus='']
srun: error: hkn0406: task 0: Exited with exit code 1

============================= JOB FEEDBACK =============================

Job ID: 3520775
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 1)
Partition: accelerated
Nodes: 1
Cores per node: 6
Nodelist: hkn0406
CPU Utilized: 00:00:10
CPU Efficiency: 4.07% of 00:04:06 core-walltime
Job Wall-clock time: 00:00:41
Starttime: Thu Sep 25 15:58:44 2025
Endtime: Thu Sep 25 15:59:25 2025
Memory Utilized: 1.64 GB
Memory Efficiency: 1.35% of 121.46 GB (121.46 GB/node)
Energy Consumed: 16062 Joule / 4.46166666666667 Watthours
Average node power draw: 391.756097560976 Watt