[tab] logs/logs_training_dyn/sample_knoms_overfit_10_3292064.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=0:45:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_dyn/%x_%j.log
#SBATCH --error=logs/logs_training_dyn/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=sample_knoms_overfit_10
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

# data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'
# checkpoint_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/dyn/3282489/genie_1750610596_12500'
# seq_len=16
# output_dir='sample_results/knoms/'

# python sample.py --checkpoint $checkpoint_dir --data_dir $data_dir --batch_size 12 --seq_len $seq_len --output_dir $output_dir
# # CUDA_VISIBLE_DEVICES=0 python sample.py --checkpoint $checkpoint_dir --data_dir $data_dir --batch_size 12

## overfit tfrecord 10
data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards/tfrecords_10'
checkpoint_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/dyn/3289577/genie_1750786631_12500'
seq_len=16
output_dir='sample_results/knoms_overfit_10/'
mkdir -p $output_dir

python sample.py --checkpoint $checkpoint_dir --data_dir $data_dir --batch_size 12 --seq_len $seq_len --output_dir $output_dir

2025-06-25 18:21:02.190597: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750868462.483590 3749385 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750868462.522401 3749385 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1750868463.089230 3749385 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750868463.089265 3749385 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750868463.089270 3749385 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750868463.089274 3749385 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
╭─ Unrecognized options ──────────────────╮
│ Unrecognized options: --output-dir      │
│ ─────────────────────────────────────── │
│ For full helptext, run sample.py --help │
╰─────────────────────────────────────────╯

============================= JOB FEEDBACK =============================

Job ID: 3292064
Cluster: hk
User/Group: tum_ind3695/hk-project-pai00039
Account: hk-project-p0023960
State: FAILED (exit code 2)
Partition: accelerated
Nodes: 1
Cores per node: 32
Nodelist: hkn0416
CPU Utilized: 00:00:06
CPU Efficiency: 0.32% of 00:31:28 core-walltime
Job Wall-clock time: 00:00:59
Starttime: Wed Jun 25 18:20:22 2025
Endtime: Wed Jun 25 18:21:21 2025
Memory Utilized: 436.23 MB
Memory Efficiency: 0.00% of 0.00 MB
Energy Consumed: 22350 Joule / 6.20833333333333 Watthours
Average node power draw: 378.813559322034 Watt