[tab] /home/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/logs/logs_training/train_tokenizer_batch_size_scaling_1_node_3313570.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_1_node

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --min_lr=3e-4 \
    --max_lr=3e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-1-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 1-node \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
/var/spool/slurmd/job3313570/slurm_script: line 18: .venv/bin/activate: No such file or directory
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4
SLURM_JOB_UID=999226
SLURM_TASK_PID=1989439
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs
SLURMD_NODENAME=hkn0512
SLURM_JOB_START_TIME=1751511828
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1751565828
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=1
SLURM_JOBID=3313570
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=4
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e10.hkn0512
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn0512
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=4
SLURM_NNODES=1
SLURM_SUBMIT_HOST=hkn1991.localdomain
SLURM_JOB_ID=3313570
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_tokenizer_batch_size_scaling_1_node
SLURM_NTASKS_PER_NODE=4
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn0512
GpuFreq=control_disabled
2025-07-03 05:04:39.228831: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-03 05:04:39.228830: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-03 05:04:39.228828: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-03 05:04:39.228830: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751511879.311058 1989542 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751511879.311221 1989540 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751511879.311363 1989541 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751511879.311333 1989543 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1751511879.354914 1989540 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1751511879.354954 1989541 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1751511879.354924 1989542 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1751511879.354940 1989543 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1751511879.676045 1989540 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676076 1989540 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676078 1989540 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676080 1989540 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676047 1989541 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676081 1989541 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676084 1989541 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676085 1989541 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676048 1989542 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676085 1989542 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676087 1989542 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676089 1989542 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676050 1989543 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676086 1989543 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676088 1989543 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511879.676089 1989543 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751511910.121814 1989542 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1751511910.121818 1989543 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1751511910.121838 1989540 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1751511910.121987 1989541 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.19.11
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/wandb/run-20250703_050542-9ohe3ypl
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run tokenizer-batch-size-scaling-1-node-3313570
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/9ohe3ypl
Running on 4 devices.
Counting all components: ['encoder', 'vq', 'decoder']
Parameter counts:
{'encoder': 18978432, 'vq': 32768, 'decoder': 18978416, 'total': 37989616}
Running on 4 devices.
Counting all components: ['encoder', 'vq', 'decoder']
Parameter counts:
{'encoder': 18978432, 'vq': 32768, 'decoder': 18978416, 'total': 37989616}
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_tokenizer.py", line 213, in <module>
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_tokenizer.py", line 213, in <module>
    for x in os.listdir(args.data_dir)
FileNotFoundError: [Errno 2] No such file or directory: 'hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord'
    for x in os.listdir(args.data_dir)
FileNotFoundError: [Errno 2] No such file or directory: 'hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord'
Running on 4 devices.
Counting all components: ['encoder', 'vq', 'decoder']
Parameter counts:
{'encoder': 18978432, 'vq': 32768, 'decoder': 18978416, 'total': 37989616}
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_tokenizer.py", line 213, in <module>
    for x in os.listdir(args.data_dir)
FileNotFoundError: [Errno 2] No such file or directory: 'hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord'
Running on 4 devices.
Counting all components: ['encoder', 'vq', 'decoder']
Parameter counts:
{'encoder': 18978432, 'vq': 32768, 'decoder': 18978416, 'total': 37989616}
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_tokenizer.py", line 213, in <module>
    for x in os.listdir(args.data_dir)
FileNotFoundError: [Errno 2] No such file or directory: 'hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord'
wandb: 
wandb: 🚀 View run tokenizer-batch-size-scaling-1-node-3313570 at: https://wandb.ai/instant-uv/jafar/runs/9ohe3ypl
wandb: Find logs at: ../../../../../hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/wandb/run-20250703_050542-9ohe3ypl/logs
srun: error: hkn0512: tasks 0,2-3: Exited with exit code 1
srun: error: hkn0512: task 1: Exited with exit code 1

============================= JOB FEEDBACK =============================

Job ID: 3313570
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 1)
Partition: accelerated
Nodes: 1
Cores per node: 24
Nodelist: hkn0512
CPU Utilized: 00:05:01
CPU Efficiency: 10.11% of 00:49:36 core-walltime
Job Wall-clock time: 00:02:04
Starttime: Thu Jul  3 05:03:48 2025
Endtime: Thu Jul  3 05:05:52 2025
Memory Utilized: 6.66 GB (estimated maximum)
Memory Efficiency: 0.00% of 0.00 MB (0.00 MB/node)
Energy Consumed: 128622 Joule / 35.7283333333333 Watthours
Average node power draw: 1037.27419354839 Watt