[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/train_tokenizer_default_single_node_3543642.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_default_single_node

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m/val


job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python jasmine/train_tokenizer.py \
    --save_ckpt \
    --image_height=240 \
    --image_width=320 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=32 \
    --init_lr=0 \
    --log \
    --name=doom-tokenizer-default-1node-$slurm_job_id \
    --tags tokenizer doom default \
    --entity instant-uv \
    --project jafar \
    --data_dir $array_records_dir_train \
    --val_data_dir $array_records_dir_valSLURM_STEP_NUM_TASKS=1
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4
SLURM_JOB_UID=999226
SLURM_TASK_PID=2970754
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine
SLURMD_NODENAME=hkn0601
SLURM_JOB_START_TIME=1759675437
SLURM_STEP_NODELIST=hkn0401
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1759693437
SLURM_PMI2_SRUN_PORT=37977
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=1
SLURM_STEPID=4294967290
SLURM_JOBID=3543642
SLURM_PTY_PORT=35539
SLURM_JOB_QOS=normal
SLURM_LAUNCH_NODE_IPADDR=10.0.7.199
SLURM_PTY_WIN_ROW=23
SLURM_PMI2_PROC_MAPPING=(vector,(0,1,1))
SLURMD_DEBUG=2
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=4
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e29.hkn0601
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SRUN_COMM_HOST=10.0.7.199
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_PTY_WIN_COL=136
SLURM_NODELIST=hkn0601
SLURM_SRUN_COMM_PORT=40819
SLURM_STEP_ID=4294967290
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=4
SLURM_NNODES=1
SLURM_SUBMIT_HOST=hkn0401.localdomain
SLURM_JOB_ID=3543642
SLURM_NODEID=0
SLURM_STEP_NUM_NODES=1
SLURM_STEP_TASKS_PER_NODE=1
SLURM_MPI_TYPE=pmi2
SLURM_PMI2_STEP_NODES=hkn0401
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_tokenizer_default_single_node
SLURM_NTASKS_PER_NODE=4
SLURM_STEP_LAUNCHER_PORT=40819
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn0601
GpuFreq=control_disabled
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.22.0
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/wandb/run-20251005_164445-oxurw67i
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run doom-tokenizer-default-1node-3543642
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/oxurw67i
Running on 4 devices.
Counting all components: ['decoder', 'encoder', 'vq']
Parameter counts:
{'decoder': 16858736, 'encoder': 16858752, 'vq': 32768, 'total': 33750256}
Running on 4 devices.
Counting all components: ['decoder', 'encoder', 'vq']
Parameter counts:
{'decoder': 16858736, 'encoder': 16858752, 'vq': 32768, 'total': 33750256}
Starting training from step 0...
Running on 4 devices.
Counting all components: ['decoder', 'encoder', 'vq']
Parameter counts:
{'decoder': 16858736, 'encoder': 16858752, 'vq': 32768, 'total': 33750256}
Starting training from step 0...
Running on 4 devices.
Counting all components: ['decoder', 'encoder', 'vq']
Parameter counts:
{'decoder': 16858736, 'encoder': 16858752, 'vq': 32768, 'total': 33750256}
Starting training from step 0...
Total memory size: 31.3 GB, Output size: 0.5 GB, Temp size: 30.8 GB, Argument size: 0.4 GB, Host temp size: 0.0 GB.
FLOPs: 2.192e+13, Bytes: 9.699e+11 (903.3 GB), Intensity: 22.6 FLOPs/byte
Starting training from step 0...

Memstats: After params initialized:
	Using (GB) 0.62 / 35.55 (1.744023%) on cuda:0
Traceback (most recent call last):
  File "/usr/lib64/python3.12/shutil.py", line 874, in move
    os.rename(src, real_dst)
OSError: [Errno 18] Invalid cross-device link: '/scratch/slurm_tmpdir/job_3543642/tmprk32brkiwandb-media/enlmfabp.png' -> '/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/wandb/run-20251005_164445-oxurw67i/files/media/images/true_vs_recon_407_beedeb9e64f7b0d3bde7.png'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/jasmine/train_tokenizer.py", line 574, in <module>
    main(args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/jasmine/train_tokenizer.py", line 541, in main
    wandb.log(log_images)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper
    return func(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn
    return func(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/wandb_run.py", line 444, in wrapper
    return func(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/wandb_run.py", line 2028, in log
    self._log(data=data, step=step, commit=commit)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1739, in _log
    self._partial_history_callback(data, step, commit)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper
    return func(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1566, in _partial_history_callback
    self._backend.interface.publish_partial_history(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/interface/interface.py", line 682, in publish_partial_history
    data = history_dict_to_json(run, data, step=user_step, ignore_copy_err=True)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/data_types/utils.py", line 54, in history_dict_to_json
    payload[key] = val_to_json(
                   ^^^^^^^^^^^^
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/data_types/utils.py", line 158, in val_to_json
    val.bind_to_run(run, key, namespace)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/data_types/image.py", line 519, in bind_to_run
    super().bind_to_run(run, key, step, id_, ignore_copy_err=ignore_copy_err)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib64/python3.12/site-packages/wandb/sdk/data_types/base_types/media.py", line 146, in bind_to_run
    shutil.move(self._path, new_path)
  File "/usr/lib64/python3.12/shutil.py", line 894, in move
    copy_function(src, real_dst)
  File "/usr/lib64/python3.12/shutil.py", line 463, in copy2
    copyfile(src, dst, follow_symlinks=follow_symlinks)
  File "/usr/lib64/python3.12/shutil.py", line 262, in copyfile
    with open(dst, 'wb') as fdst:
OSError: [Errno 122] Disk quota exceeded
wandb: 
wandb: 🚀 View run doom-tokenizer-default-1node-3543642 at: https://wandb.ai/instant-uv/jafar/runs/oxurw67i
wandb: Find logs at: ../../../../../hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/wandb/run-20251005_164445-oxurw67i/logs
E1005 20:47:45.724894 2970850 coordination_service.cc:1852] Shutdown barrier in coordination service has failed:
DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::11443187686619195905::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 1/4.
The first task at the barrier: /job:jax_worker/replica:0/task:0. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::11443187686619195905']
This suggests that the workers are out of sync. Either at least one worker (a) crashed early due to program error or scheduler events (e.g. preemption, eviction), (b) was too fast in its execution, or (c) too slow / hanging. Check the logs (both the program and scheduler events) for an earlier error to identify the root cause.
E1005 20:47:45.724922 2970850 coordination_service.cc:1893] Use error polling to propagate the following error to all tasks: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::11443187686619195905::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 1/4.
The first task at the barrier: /job:jax_worker/replica:0/task:0. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::11443187686619195905'] [type.googleapis.com/tensorflow.CoordinationServiceError='']
E1005 20:47:45.725141 2970926 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E1005 20:47:45.725189 2970929 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E1005 20:47:45.725409 2970892 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
E1005 20:47:45.725573 2970925 coordination_service_agent.cc:332] Polled an error from coordination service (this can be an error from this or another task).
F1005 20:47:45.732621 2970926 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::11443187686619195905::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 1/4.
The first task at the barrier: /job:jax_worker/replica:0/task:0. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::11443187686619195905']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F1005 20:47:45.732940 2970925 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::11443187686619195905::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 1/4.
The first task at the barrier: /job:jax_worker/replica:0/task:0. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::11443187686619195905']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F1005 20:47:45.732953 2970892 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::11443187686619195905::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 1/4.
The first task at the barrier: /job:jax_worker/replica:0/task:0. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::11443187686619195905']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
F1005 20:47:45.733296 2970929 client.h:75] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::11443187686619195905::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 1/4.
The first task at the barrier: /job:jax_worker/replica:0/task:0. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::11443187686619195905']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
/usr/lib64/python3.12/multiprocessing/resource_tracker.py:254: UserWarning: resource_tracker: There appear to be 20 leaked shared_memory objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
srun: error: hkn0601: tasks 1-2: Exited with exit code 1
srun: error: hkn0601: task 0: Exited with exit code 1
srun: error: hkn0601: task 3: Exited with exit code 1

============================= JOB FEEDBACK =============================

Job ID: 3543642
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 1)
Partition: accelerated
Nodes: 1
Cores per node: 24
Nodelist: hkn0601
CPU Utilized: 14:01:04
CPU Efficiency: 14.37% of 4-01:32:00 core-walltime
Job Wall-clock time: 04:03:50
Starttime: Sun Oct  5 16:43:57 2025
Endtime: Sun Oct  5 20:47:47 2025
Memory Utilized: 27.92 GB
Memory Efficiency: 5.75% of 485.84 GB (485.84 GB/node)
The task which had the largest memory consumption differs by 102.21% from the average task max memory consumption
Energy Consumed: 20101103 Joule / 5583.63972222222 Watthours
Average node power draw: 1373.96466165414 Watt