[tab] logs_training/train_dynamics_minecraft_overfit_sample_1.5M_3297569.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=10:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --mail-user=mihir.mahajan2002@gmail.com
#SBATCH --job-name=train_dynamics_minecraft_overfit_sample_1.5M
#SBATCH --mem=50G
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

echo Running dynamics model overfit run (tiny model A, ~1.5M params). Slurm id: $slurm_job_id

tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3290391/tokenizer_1750845012_50000/
lam_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3290392/lam_1750845133_180000/

srun python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=5e-7 \
    --max_lr=5e-6 \
    --warmup_steps=125 \
    --log_image_interval=100 \
    --log \
    --name=dynamics-model-size-scaling-1.5M-$slurm_job_id \
    --tags dynamics model-size-scaling 1.5M tiny A \
    --log_checkpoint_interval=500 \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --lam_checkpoint=$lam_ckpt_dir \
    --data_dir $tf_records_dir \
    --dyna_dim=128 \
    --dyna_num_blocks=2 \
    --dyna_num_heads=4/var/spool/slurmd/job3297569/slurm_script: line 21: .venv/bin/activate: No such file or directory
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=1
SLURM_JOB_UID=999226
SLURM_TASK_PID=226766
SLURM_JOB_GPUS=2
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs
SLURMD_NODENAME=hkn0505
SLURM_JOB_START_TIME=1750996964
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1751032964
SLURM_CPUS_ON_NODE=6
SLURM_JOB_CPUS_PER_NODE=6
SLURM_GPUS_ON_NODE=1
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=1
SLURM_JOBID=3297569
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=1
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e10.hkn0505
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_MEM_PER_NODE=51200
SLURM_NODELIST=hkn0505
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=1
SLURM_NNODES=1
SLURM_SUBMIT_HOST=hkn1990.localdomain
SLURM_JOB_ID=3297569
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dynamics_minecraft_overfit_sample_1.5M
SLURM_NTASKS_PER_NODE=1
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn0505
/var/spool/slurmd/job3297569/slurm_script: line 34: syntax error near unexpected token `('
/var/spool/slurmd/job3297569/slurm_script: line 34: `echo Running dynamics model overfit run (tiny model A, ~1.5M params). Slurm id: $slurm_job_id'

============================= JOB FEEDBACK =============================

Job ID: 3297569
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 2)
Partition: accelerated
Nodes: 1
Cores per node: 6
Nodelist: hkn0505
CPU Utilized: 00:00:00
CPU Efficiency: 0.00% of 00:02:54 core-walltime
Job Wall-clock time: 00:00:29
Starttime: Fri Jun 27 06:02:44 2025
Endtime: Fri Jun 27 06:03:13 2025
Memory Utilized: 10.07 MB
Memory Efficiency: 0.02% of 50.00 GB
Energy Consumed: 6280 Joule / 1.74444444444444 Watthours
Average node power draw: 216.551724137931 Watt