[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/big-runs/dynamics-cotraining-modelsize-scaling/train_dynamics_modelsize_scaling_500M_32_node_3335748.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=32
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/big-runs/dynamics-cotraining-modelsize-scaling/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/big-runs/dynamics-cotraining-modelsize-scaling/%x_%j.log
#SBATCH --job-name=train_dynamics_modelsize_scaling_500M_32_node

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/dynamics-cotraining-modelsize-scaling/$job_name
mkdir -p $CHECKPOINT_DIR

tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/train_tokenizer_batch_size_scaling_16_node/3321526/tokenizer_22000/

env | grep SLURM

srun python train_dynamics.py \
    --save_ckpt \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=1.5e-5 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=dynamics-modelsize-scaling-500M-$slurm_job_id \
    --tags dynamics modelsize-scaling 500M \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir \
    --dyna_dim=1536 \
    --dyna_num_blocks=24 \
    --dyna_num_heads=24
/var/spool/slurmd/job3335748/slurm_script: line 18: .venv/bin/activate: No such file or directory
SLURM_STEP_NUM_TASKS=1
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x32)
SLURM_JOB_UID=999226
SLURM_TASK_PID=1091557
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs
SLURMD_NODENAME=hkn0409
SLURM_JOB_START_TIME=1752249626
SLURM_STEP_NODELIST=hkn0410
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1752422426
SLURM_PMI2_SRUN_PORT=41685
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x32)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=32
SLURM_STEPID=4294967290
SLURM_JOBID=3335748
SLURM_PTY_PORT=35723
SLURM_JOB_QOS=normal
SLURM_LAUNCH_NODE_IPADDR=10.0.7.201
SLURM_PTY_WIN_ROW=48
SLURM_PMI2_PROC_MAPPING=(vector,(0,1,1))
SLURMD_DEBUG=2
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=128
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e12.hkn0409
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SRUN_COMM_HOST=10.0.7.201
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_PTY_WIN_COL=228
SLURM_NODELIST=hkn[0409,0412-0413,0416,0418-0419,0421,0423,0503,0511-0513,0516,0518-0521,0527-0528,0531,0533-0534,0702,0704,0712-0714,0719,0725,0729,0731,0735]
SLURM_SRUN_COMM_PORT=32987
SLURM_STEP_ID=4294967290
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=128
SLURM_NNODES=32
SLURM_SUBMIT_HOST=hkn0410.localdomain
SLURM_JOB_ID=3335748
SLURM_NODEID=0
SLURM_STEP_NUM_NODES=1
SLURM_STEP_TASKS_PER_NODE=1
SLURM_MPI_TYPE=pmi2
SLURM_PMI2_STEP_NODES=hkn0410
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_dynamics_modelsize_scaling_500M_32_node
SLURM_NTASKS_PER_NODE=4
SLURM_STEP_LAUNCHER_PORT=32987
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0409,0412-0413,0416,0418-0419,0421,0423,0503,0511-0513,0516,0518-0521,0527-0528,0531,0533-0534,0702,0704,0712-0714,0719,0725,0729,0731,0735]
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
Running on 128 devices.
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
    raise ValueError(
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Running on 128 devices.
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
    raise ValueError(
Running on 128 devices.
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_dynamics.py", line 125, in <module>
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
    raise ValueError(
ValueError: Global batch size 96 must be divisible by number of devices 128.
srun: error: hkn0421: task 25: Exited with exit code 1
srun: error: hkn0702: tasks 88-89: Exited with exit code 1
srun: error: hkn0518: tasks 52-53: Exited with exit code 1
srun: error: hkn0520: tasks 61,63: Exited with exit code 1
srun: error: hkn0713: tasks 102-103: Exited with exit code 1
srun: error: hkn0421: tasks 24,27: Exited with exit code 1
srun: error: hkn0702: task 90: Exited with exit code 1
srun: error: hkn0518: task 54: Exited with exit code 1
srun: error: hkn0520: task 62: Exited with exit code 1
srun: error: hkn0713: task 101: Exited with exit code 1
srun: error: hkn0503: tasks 33-34: Exited with exit code 1
srun: error: hkn0519: task 57: Exited with exit code 1
srun: error: hkn0409: tasks 1,3: Exited with exit code 1
srun: error: hkn0714: task 105: Exited with exit code 1
srun: error: hkn0516: tasks 48,51: Exited with exit code 1
srun: error: hkn0419: tasks 20,22: Exited with exit code 1
srun: error: hkn0423: tasks 29,31: Exited with exit code 1
srun: error: hkn0528: tasks 72-73: Exited with exit code 1
srun: error: hkn0719: tasks 108,110: Exited with exit code 1
srun: error: hkn0511: tasks 37-38: Exited with exit code 1
srun: error: hkn0513: tasks 45-46: Exited with exit code 1
srun: error: hkn0735: tasks 125,127: Exited with exit code 1
srun: error: hkn0412: tasks 4,6: Exited with exit code 1
srun: error: hkn0533: tasks 82-83: Exited with exit code 1
srun: error: hkn0731: tasks 120,122: Exited with exit code 1
srun: error: hkn0725: tasks 112,114: Exited with exit code 1
srun: error: hkn0729: tasks 116,118: Exited with exit code 1
srun: error: hkn0413: tasks 8,10: Exited with exit code 1
srun: error: hkn0521: tasks 64,66: Exited with exit code 1
srun: error: hkn0418: tasks 17,19: Exited with exit code 1
srun: error: hkn0527: tasks 68,71: Exited with exit code 1
srun: error: hkn0531: tasks 77,79: Exited with exit code 1
srun: error: hkn0416: tasks 12,15: Exited with exit code 1
srun: error: hkn0534: tasks 85-86: Exited with exit code 1
srun: error: hkn0702: task 91: Exited with exit code 1
srun: error: hkn0421: task 26: Exited with exit code 1
srun: error: hkn0712: tasks 96,98: Exited with exit code 1
srun: error: hkn0512: tasks 41-42: Exited with exit code 1
srun: error: hkn0704: tasks 92,94: Exited with exit code 1
srun: error: hkn0519: tasks 56,59: Exited with exit code 1
srun: error: hkn0520: task 60: Exited with exit code 1
srun: error: hkn0518: task 55: Exited with exit code 1
srun: error: hkn0714: tasks 106-107: Exited with exit code 1
srun: error: hkn0528: task 74: Exited with exit code 1
srun: error: hkn0735: task 126: Exited with exit code 1
srun: error: hkn0409: task 0: Exited with exit code 1
srun: error: hkn0713: task 100: Exited with exit code 1
srun: error: hkn0731: task 121: Exited with exit code 1
srun: error: hkn0419: task 21: Exited with exit code 1
srun: error: hkn0423: task 28: Exited with exit code 1
srun: error: hkn0719: task 111: Exited with exit code 1
srun: error: hkn0521: task 65: Exited with exit code 1
srun: error: hkn0413: task 9: Exited with exit code 1
srun: error: hkn0516: task 49: Exited with exit code 1
srun: error: hkn0412: task 5: Exited with exit code 1
srun: error: hkn0503: task 32: Exited with exit code 1
srun: error: hkn0513: task 47: Exited with exit code 1
srun: error: hkn0527: task 70: Exited with exit code 1
srun: error: hkn0533: task 80: Exited with exit code 1
srun: error: hkn0531: task 76: Exited with exit code 1
srun: error: hkn0725: task 115: Exited with exit code 1
srun: error: hkn0416: task 14: Exited with exit code 1
srun: error: hkn0729: task 117: Exited with exit code 1
srun: error: hkn0534: task 84: Exited with exit code 1
srun: error: hkn0418: task 16: Exited with exit code 1
srun: error: hkn0712: task 97: Exited with exit code 1
srun: error: hkn0512: task 40: Exited with exit code 1
srun: error: hkn0704: task 93: Exited with exit code 1
srun: error: hkn0511: task 36: Exited with exit code 1
srun: error: hkn0503: task 35: Exited with exit code 1
srun: error: hkn0516: task 50: Exited with exit code 1
srun: error: hkn0409: task 2: Exited with exit code 1
srun: error: hkn0731: task 123: Exited with exit code 1
srun: error: hkn0423: task 30: Exited with exit code 1
srun: error: hkn0729: task 119: Exited with exit code 1
srun: error: hkn0528: task 75: Exited with exit code 1
srun: error: hkn0725: task 113: Exited with exit code 1
srun: error: hkn0719: task 109: Exited with exit code 1
srun: error: hkn0519: task 58: Exited with exit code 1
srun: error: hkn0412: task 7: Exited with exit code 1
srun: error: hkn0735: task 124: Exited with exit code 1
srun: error: hkn0513: task 44: Exited with exit code 1
srun: error: hkn0714: task 104: Exited with exit code 1
srun: error: hkn0527: task 69: Exited with exit code 1
srun: error: hkn0533: task 81: Exited with exit code 1
srun: error: hkn0521: task 67: Exited with exit code 1
srun: error: hkn0416: task 13: Exited with exit code 1
srun: error: hkn0413: task 11: Exited with exit code 1
srun: error: hkn0418: task 18: Exited with exit code 1
srun: error: hkn0534: task 87: Exited with exit code 1
srun: error: hkn0531: task 78: Exited with exit code 1
srun: error: hkn0512: task 43: Exited with exit code 1
srun: error: hkn0712: task 99: Exited with exit code 1
srun: error: hkn0704: task 95: Exited with exit code 1
srun: error: hkn0511: task 39: Exited with exit code 1
srun: error: hkn0419: task 23: Exited with exit code 1

============================= JOB FEEDBACK =============================

Job ID: 3335748
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 1)
Partition: accelerated
Nodes: 32
Cores per node: 24
Nodelist: hkn[0409,0412-0413,0416,0418-0419,0421,0423,0503,0511-0513,0516,0518-0521,0527-0528,0531,0533-0534,0702,0704,0712-0714,0719,0725,0729,0731,0735]
CPU Utilized: 00:09:16
CPU Efficiency: 1.34% of 11:31:12 core-walltime
Job Wall-clock time: 00:00:54
Starttime: Fri Jul 11 18:00:26 2025
Endtime: Fri Jul 11 18:01:20 2025
Memory Utilized: 32.08 GB (estimated maximum)
Memory Efficiency: 0.00% of 0.00 MB (0.00 MB/node)
Energy Consumed: 33462963 Joule / 9295.2675 Watthours
Average node power draw: 619684.5 Watt