[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/dynamics/causal/train_dynamics_causal_1_node_3415690.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/dynamics/causal/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/dynamics/causal/%x_%j.log
#SBATCH --job-name=train_dynamics_causal_1_node
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi



# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/array_records

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/coinrun/causal/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

tokenizer_ckpt_dir="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046"

env | grep SLURM

srun python train_dynamics.py \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --dyna_type=causal \
    --init_lr=0 \
    --max_lr=1e-5 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=coinrun-dynamics-causal-1-node-$slurm_job_id \
    --tags coinrun dynamics causal 1-node \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
/var/spool/slurmd/job3415690/slurm_script: line 42: .venv/bin/activate: No such file or directory
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4
SLURM_JOB_UID=999226
SLURM_TASK_PID=1852307
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs
SLURMD_NODENAME=hkn0631
SLURM_JOB_START_TIME=1754942565
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1755115365
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=1
SLURM_JOBID=3415690
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=4
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e30.hkn0631
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn0631
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=4
SLURM_NNODES=1
SLURM_SUBMIT_HOST=hkn1993.localdomain
SLURM_JOB_ID=3415690
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dynamics_causal_1_node
SLURM_NTASKS_PER_NODE=4
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn0631
GpuFreq=control_disabled
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.19.11
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/wandb/run-20250811_220351-3415690
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run coinrun-dynamics-causal-1-node-3415690
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/3415690
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 60000
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/060000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 60000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/060000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/040000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 40000
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/040000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 80000
WARNING:absl:Missing metrics for step 80000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/080000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/080000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 20000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/020000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 130000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/130000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 130000
WARNING:absl:Missing metrics for step 130000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/130000/metrics/metrics not found.
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/130000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 130000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/130000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 131000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/131000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 131000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/131000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 131000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/131000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 120000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/120000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 131000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/131000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 100000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/100000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 129000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/129000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 129000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/129000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 129000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/129000/metrics/metrics not found.
WARNING:absl:Missing metrics for step 129000
ERROR:absl:File /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3414046/129000/metrics/metrics not found.
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1251: UserWarning: Sharding info not provided when restoring. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.
  warnings.warn(
Running on 4 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 26555392, 'lam': 35115232, 'tokenizer': 33750256, 'total': 95420880}
Starting training from step 0...
Running on 4 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 26555392, 'lam': 35115232, 'tokenizer': 33750256, 'total': 95420880}
Starting training from step 0...
Running on 4 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 26555392, 'lam': 35115232, 'tokenizer': 33750256, 'total': 95420880}
Starting training from step 0...
Running on 4 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 26555392, 'lam': 35115232, 'tokenizer': 33750256, 'total': 95420880}
Starting training from step 0...
ERROR:absl:Error occurred in child process with worker_index: 0
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 250, in _worker_loop
    next_element = next(element_producer)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 411, in __call__
    yield from self._read_and_transform_data(last_seen_index)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 596, in _apply_transform
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 586, in _apply_transform
    for r in batch_op(input_iterator):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/operations.py", line 152, in __call__
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 602, in _apply_transform
    raise e
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 598, in _apply_transform
    output_record, filter_result = fn(input_record)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 568, in <lambda>
    record.Record(r.metadata, transform.random_map(r.data, r.metadata.rng)),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/utils/dataloader.py", line 80, in random_map
    episode_tensor = episode_tensor.reshape(video_shape)
ValueError: cannot reshape array of size 6193152 into shape (504,90,160,3)
ERROR:absl:Error occurred in child process with worker_index: 0
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 250, in _worker_loop
    next_element = next(element_producer)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 411, in __call__
    yield from self._read_and_transform_data(last_seen_index)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 596, in _apply_transform
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 586, in _apply_transform
    for r in batch_op(input_iterator):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/operations.py", line 152, in __call__
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 602, in _apply_transform
    raise e
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 598, in _apply_transform
    output_record, filter_result = fn(input_record)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 568, in <lambda>
    record.Record(r.metadata, transform.random_map(r.data, r.metadata.rng)),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/utils/dataloader.py", line 80, in random_map
    episode_tensor = episode_tensor.reshape(video_shape)
ValueError: cannot reshape array of size 12288000 into shape (1000,90,160,3)
ERROR:absl:Processing Failed. Shutting down.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 361, in <module>
    for videos in dataloader:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 355, in <genexpr>
    dataloader = (
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 479, in __next__
    result_record = next(self._iterator)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 380, in _iterator_with_context
    yield from it
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 784, in __next__
    raise element
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 616, in _process_elements_in_grain_pool
    for element in g_pool:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 498, in __next__
    raise self.worker_error_queue.get(timeout=_QUEUE_WAIT_TIMEOUT)
grain._src.python.grain_pool.RemoteWorkerError: Grain worker 0 failed with the following error:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 250, in _worker_loop
    next_element = next(element_producer)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 411, in __call__
    yield from self._read_and_transform_data(last_seen_index)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 596, in _apply_transform
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 586, in _apply_transform
    for r in batch_op(input_iterator):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/operations.py", line 152, in __call__
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 602, in _apply_transform
    raise e
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 598, in _apply_transform
    output_record, filter_result = fn(input_record)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 568, in <lambda>
    record.Record(r.metadata, transform.random_map(r.data, r.metadata.rng)),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/utils/dataloader.py", line 80, in random_map
    episode_tensor = episode_tensor.reshape(video_shape)
ValueError: cannot reshape array of size 12288000 into shape (1000,90,160,3)

ERROR:absl:Processing Failed. Shutting down.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 361, in <module>
    for videos in dataloader:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 355, in <genexpr>
    dataloader = (
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 479, in __next__
    result_record = next(self._iterator)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 380, in _iterator_with_context
    yield from it
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 784, in __next__
    raise element
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 616, in _process_elements_in_grain_pool
    for element in g_pool:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 498, in __next__
    raise self.worker_error_queue.get(timeout=_QUEUE_WAIT_TIMEOUT)
grain._src.python.grain_pool.RemoteWorkerError: Grain worker 0 failed with the following error:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 250, in _worker_loop
    next_element = next(element_producer)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 411, in __call__
    yield from self._read_and_transform_data(last_seen_index)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 596, in _apply_transform
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 586, in _apply_transform
    for r in batch_op(input_iterator):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/operations.py", line 152, in __call__
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 602, in _apply_transform
    raise e
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 598, in _apply_transform
    output_record, filter_result = fn(input_record)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 568, in <lambda>
    record.Record(r.metadata, transform.random_map(r.data, r.metadata.rng)),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/utils/dataloader.py", line 80, in random_map
    episode_tensor = episode_tensor.reshape(video_shape)
ValueError: cannot reshape array of size 6193152 into shape (504,90,160,3)

ERROR:absl:Error occurred in child process with worker_index: 2
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 250, in _worker_loop
    next_element = next(element_producer)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 411, in __call__
    yield from self._read_and_transform_data(last_seen_index)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 596, in _apply_transform
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 586, in _apply_transform
    for r in batch_op(input_iterator):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/operations.py", line 152, in __call__
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 602, in _apply_transform
    raise e
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 598, in _apply_transform
    output_record, filter_result = fn(input_record)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 568, in <lambda>
    record.Record(r.metadata, transform.random_map(r.data, r.metadata.rng)),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/utils/dataloader.py", line 80, in random_map
    episode_tensor = episode_tensor.reshape(video_shape)
ValueError: cannot reshape array of size 8933376 into shape (727,90,160,3)
ERROR:absl:Error occurred in child process with worker_index: 4
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 250, in _worker_loop
    next_element = next(element_producer)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 411, in __call__
    yield from self._read_and_transform_data(last_seen_index)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 596, in _apply_transform
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 586, in _apply_transform
    for r in batch_op(input_iterator):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/operations.py", line 152, in __call__
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 602, in _apply_transform
    raise e
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 598, in _apply_transform
    output_record, filter_result = fn(input_record)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 568, in <lambda>
    record.Record(r.metadata, transform.random_map(r.data, r.metadata.rng)),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/utils/dataloader.py", line 80, in random_map
    episode_tensor = episode_tensor.reshape(video_shape)
ValueError: cannot reshape array of size 5541888 into shape (451,90,160,3)
ERROR:absl:Error occurred in child process with worker_index: 0
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 250, in _worker_loop
    next_element = next(element_producer)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 411, in __call__
    yield from self._read_and_transform_data(last_seen_index)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 596, in _apply_transform
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 586, in _apply_transform
    for r in batch_op(input_iterator):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/operations.py", line 152, in __call__
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 602, in _apply_transform
    raise e
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 598, in _apply_transform
    output_record, filter_result = fn(input_record)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 568, in <lambda>
    record.Record(r.metadata, transform.random_map(r.data, r.metadata.rng)),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/utils/dataloader.py", line 80, in random_map
    episode_tensor = episode_tensor.reshape(video_shape)
ValueError: cannot reshape array of size 2125824 into shape (173,90,160,3)
ERROR:absl:Error occurred in child process with worker_index: 2
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 250, in _worker_loop
    next_element = next(element_producer)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 411, in __call__
    yield from self._read_and_transform_data(last_seen_index)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 596, in _apply_transform
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 586, in _apply_transform
    for r in batch_op(input_iterator):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/operations.py", line 152, in __call__
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 602, in _apply_transform
    raise e
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 598, in _apply_transform
    output_record, filter_result = fn(input_record)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 568, in <lambda>
    record.Record(r.metadata, transform.random_map(r.data, r.metadata.rng)),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/utils/dataloader.py", line 80, in random_map
    episode_tensor = episode_tensor.reshape(video_shape)
ValueError: cannot reshape array of size 12288000 into shape (1000,90,160,3)
ERROR:absl:Processing Failed. Shutting down.
ERROR:absl:Processing Failed. Shutting down.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 361, in <module>
    for videos in dataloader:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 355, in <genexpr>
    dataloader = (
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 479, in __next__
    result_record = next(self._iterator)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 380, in _iterator_with_context
    yield from it
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 784, in __next__
    raise element
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 616, in _process_elements_in_grain_pool
    for element in g_pool:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 498, in __next__
    raise self.worker_error_queue.get(timeout=_QUEUE_WAIT_TIMEOUT)
grain._src.python.grain_pool.RemoteWorkerError: Grain worker 2 failed with the following error:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 250, in _worker_loop
    next_element = next(element_producer)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 411, in __call__
    yield from self._read_and_transform_data(last_seen_index)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 596, in _apply_transform
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 586, in _apply_transform
    for r in batch_op(input_iterator):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/operations.py", line 152, in __call__
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 602, in _apply_transform
    raise e
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 598, in _apply_transform
    output_record, filter_result = fn(input_record)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 568, in <lambda>
    record.Record(r.metadata, transform.random_map(r.data, r.metadata.rng)),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/utils/dataloader.py", line 80, in random_map
    episode_tensor = episode_tensor.reshape(video_shape)
ValueError: cannot reshape array of size 8933376 into shape (727,90,160,3)

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 361, in <module>
    for videos in dataloader:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 355, in <genexpr>
    dataloader = (
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 479, in __next__
    result_record = next(self._iterator)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 380, in _iterator_with_context
    yield from it
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 784, in __next__
    raise element
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 616, in _process_elements_in_grain_pool
    for element in g_pool:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 498, in __next__
    raise self.worker_error_queue.get(timeout=_QUEUE_WAIT_TIMEOUT)
grain._src.python.grain_pool.RemoteWorkerError: Grain worker 2 failed with the following error:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/grain_pool.py", line 250, in _worker_loop
    next_element = next(element_producer)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 411, in __call__
    yield from self._read_and_transform_data(last_seen_index)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 596, in _apply_transform
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 586, in _apply_transform
    for r in batch_op(input_iterator):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/operations.py", line 152, in __call__
    for input_record in input_iterator:
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 602, in _apply_transform
    raise e
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 598, in _apply_transform
    output_record, filter_result = fn(input_record)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/grain/_src/python/data_loader.py", line 568, in <lambda>
    record.Record(r.metadata, transform.random_map(r.data, r.metadata.rng)),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/utils/dataloader.py", line 80, in random_map
    episode_tensor = episode_tensor.reshape(video_shape)
ValueError: cannot reshape array of size 12288000 into shape (1000,90,160,3)

wandb: 
wandb: 🚀 View run coinrun-dynamics-causal-1-node-3415690 at: https://wandb.ai/instant-uv/jafar/runs/3415690
wandb: Find logs at: ../../../../../hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/wandb/run-20250811_220351-3415690/logs
srun: error: hkn0631: task 1: Exited with exit code 1
srun: error: hkn0631: tasks 0,2: Exited with exit code 1
srun: error: hkn0631: task 3: Exited with exit code 1

============================= JOB FEEDBACK =============================

Job ID: 3415690
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 1)
Partition: accelerated
Nodes: 1
Cores per node: 24
Nodelist: hkn0631
CPU Utilized: 00:03:04
CPU Efficiency: 8.61% of 00:35:36 core-walltime
Job Wall-clock time: 00:01:29
Starttime: Mon Aug 11 22:02:45 2025
Endtime: Mon Aug 11 22:04:14 2025
Memory Utilized: 8.34 GB (estimated maximum)
Memory Efficiency: 0.00% of 0.00 MB (0.00 MB/node)
Energy Consumed: 42096 Joule / 11.6933333333333 Watthours
Average node power draw: 472.988764044944 Watt