[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/jafar_og_reproduction/dynamics/train_dynamics_coinrun_og_reproduction_3472612.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=2-00:00:00
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --partition=accelerated
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/jafar_og_reproduction/dynamics/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/jafar_og_reproduction/dynamics/%x_%j.log
#SBATCH --job-name=train_dynamics_coinrun_og_reproduction

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

tags="coinrun_og dynanmics 10m_dataset repro_mihir"

npy_records_dir="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m"

# TODO mihir: update the tokenizer and lam checkpoints
tokenizer_ckpt_dir="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/jafar_og_reproduction/tokenizer/train_tokenizer_coinrun_og_reproduction/3466286"
lam_ckpt_dir="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/jafar_og_reproduction/lam/train_lam_coinrun_og_reproduction/3466287"

CHECKPOINT_DIR="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/jafar_og_reproduction/dynamics/${job_name}/${slurm_job_id}"
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tokenizer_checkpoint="${tokenizer_ckpt_dir}" \
    --patch_size=16 \
    --lam_checkpoint="${lam_ckpt_dir}" \
    --log_image_interval=1000 \
    --log \
    --name="${job_name}_${slurm_job_id}" \
    --tags ${tags} \
    --entity instant-uv \
    --project jafar \
    --data_dir $npy_records_dir \
    --wandb_id $slurm_job_id
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=1
SLURM_JOB_UID=999226
SLURM_TASK_PID=3909444
SLURM_JOB_GPUS=3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar
SLURMD_NODENAME=hkn0804
SLURM_JOB_START_TIME=1757238205
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1757411005
SLURM_CPUS_ON_NODE=8
SLURM_JOB_CPUS_PER_NODE=8
SLURM_GPUS_ON_NODE=1
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=8
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=1
SLURM_JOBID=3472612
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=8
SLURM_NTASKS=1
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e27.hkn0804
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn0804
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=1
SLURM_NNODES=1
SLURM_SUBMIT_HOST=hkn1991.localdomain
SLURM_JOB_ID=3472612
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dynamics_coinrun_og_reproduction
SLURM_NTASKS_PER_NODE=1
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn0804
GpuFreq=control_disabled
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: creating run
wandb: Tracking run with wandb version 0.21.3
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/wandb/run-20250907_114419-3472612
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run train_dynamics_coinrun_og_reproduction_3472612
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/3472612
2025-09-07 11:44:35.257771: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-09-07 11:44:48.740866: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-09-07 11:44:52.925493: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/train_dynamics.py", line 164, in <module>
    init_params = restore_genie_components(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/genie.py", line 202, in restore_genie_components
    PyTreeCheckpointer().restore(tokenizer)["model"]["params"]["params"]
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/checkpointers/checkpointer.py", line 306, in restore
    restored = self._restore(directory, args=ckpt_args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/checkpointers/checkpointer.py", line 328, in _restore
    return self._handler.restore(directory, args=args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/pytree_checkpoint_handler.py", line 842, in restore
    structure, use_zarr3_metadata = self._get_internal_metadata(directory)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/pytree_checkpoint_handler.py", line 1000, in _get_internal_metadata
    raise FileNotFoundError(
FileNotFoundError: No structure could be identified for the checkpoint at /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/jafar_og_reproduction/tokenizer/train_tokenizer_coinrun_og_reproduction/3466286.
wandb: 
wandb: 🚀 View run train_dynamics_coinrun_og_reproduction_3472612 at: https://wandb.ai/instant-uv/jafar/runs/3472612
wandb: Find logs at: ../../../../../hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/wandb/run-20250907_114419-3472612/logs
srun: error: hkn0804: task 0: Exited with exit code 1

============================= JOB FEEDBACK =============================

Job ID: 3472612
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 1)
Partition: accelerated
Nodes: 1
Cores per node: 8
Nodelist: hkn0804
CPU Utilized: 00:01:25
CPU Efficiency: 11.42% of 00:12:24 core-walltime
Job Wall-clock time: 00:01:33
Starttime: Sun Sep  7 11:43:25 2025
Endtime: Sun Sep  7 11:44:58 2025
Memory Utilized: 1.54 GB
Memory Efficiency: 0.00% of 0.00 MB
Energy Consumed: 65527 Joule / 18.2019444444444 Watthours
Average node power draw: 704.591397849462 Watt