p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] sbatch_scripts/coinrun/train_tokenizer_coinrun.sh (shellscript)
module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate



export SLURM_NTASKS=1

# ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared'
# job_name=$SLURM_JOB_NAME
# slurm_job_id=$SLURM_JOB_ID
# data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_tmp'

# CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
# mkdir -p $CHECKPOINT_DIR

# # echo 'starting training'
# # srun echo 'hello'

# python train_tokenizer.py \
#     --ckpt_dir $CHECKPOINT_DIR \
#     --batch_size=1 \
#     --num_steps=300000\
#     --warmup_steps=10000 \
#     --log_checkpoint_interval=10000 \
#     --log \
#     --entity instant-uv \
#     --project jafar \
#     --data_dir $data_dir 


module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared'

CHECKPOINT_DIR=$ws_dir/data/checkpoints/$job_name_$slurm_job_id
LOG_DIR=$ws_dir/logs/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR
mkdir -p $LOG_DIR

data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/coinrun_tfrecords'

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --log_checkpoint_interval=1000 \
    --image_height 64 \
    --image_width 64 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[terminal_command] TERMINAL ()
/bin/python3 /hkfs/home/project/hk-project-pai00039/tum_ind3695/.cursor-server/extensions/ms-python.python-2025.6.1-linux-x64/python_files/printEnvVariablesToFile.py /hkfs/home/project/hk-project-pai00039/tum_ind3695/.cursor-server/extensions/ms-python.python-2025.6.1-linux-x64/python_files/deactivate/bash/envVars.txt
[content] slurm/README.md (markdown)
# slurm-launchpad

This repository contains Slurm job scripts and tooling for ML development on the HPC cluster. Shared by Franz, Mihir, and Alfred.

## Structure

- `jobs/` — Stable job scripts ready for production
- `dev/` — Individual folders for testing & dev scripts
- `common/` — Shared setup scripts (e.g., module loads)
- `templates/` — (Optional) Templates for sbatch generation
- `utils/` — Helpers for launching jobs or analyzing logs

## Conventions

- Logs: `%x_%j.out` format
- Promote dev → jobs/ once stable
- Use `common/` for shared SBATCH headers and setup
[content] slurm/utils/init_slurm_repo.sh (shellscript)
#!/bin/bash

# Name of the repo (change if needed)
REPO_NAME="slurm-launchpad"

# Create the main structure
mkdir -p $REPO_NAME/{jobs,common,templates,utils,logs}
mkdir -p $REPO_NAME/dev/{franz,mihir,alfred}

# Create README stubs
cat > $REPO_NAME/README.md << 'EOF'
# slurm-launchpad

This repository contains Slurm job scripts and tools for ML workflows on the HPC cluster.

## Structure

- `jobs/` — Stable, production-ready sbatch scripts
- `dev/` — Individual folders for development or experimental jobs
- `common/` — Shared setup code (env, sbatch headers)
- `templates/` — Optional Jinja2 or shell templates for job generation
- `utils/` — Helper scripts (e.g., run wrapper)
- `logs/` — Optional log output directory

Team: Franz, Mihir, Alfred
EOF

cat > $REPO_NAME/dev/README.md << 'EOF'
# dev/

This folder contains personal job scripts for development or debugging.
Each team member has a separate subfolder.

Promote useful scripts to `jobs/` after validation.
EOF

cat > $REPO_NAME/common/env_setup.sh << 'EOF'
#!/bin/bash
# Common environment setup for all jobs

module load python/3.10
source ~/envs/ml/bin/activate
EOF
chmod +x $REPO_NAME/common/env_setup.sh

cat > $REPO_NAME/common/defaults.sbatch.sh << 'EOF'
#!/bin/bash
# Common SBATCH directives (source this from your sbatch scripts)

#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --mem=16G
#SBATCH --time=01:00:00
#SBATCH --output=logs/%x_%j.out
EOF

cat > $REPO_NAME/utils/run_job.sh << 'EOF'
#!/bin/bash
# Run a job with a given sbatch file

if [ -z "$1" ]; then
  echo "Usage: ./run_job.sh <sbatch_file>"
  exit 1
fi

echo "Launching job: $1"
sbatch "$1"
EOF
chmod +x $REPO_NAME/utils/run_job.sh

echo "Slurm repo '$REPO_NAME' created!"
[tab] sbatch_scripts/procgen/data_gen_gym_coinrun.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --time=01:30:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-pai00039
#SBATCH --cpus-per-task=12
#SBATCH --output=logs_data_gen/slurm_%j.log
#SBATCH --error=logs_data_gen/slurm_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=data_gen_coinrun-test
#SBATCH --mail-type=ALL


# Log the sbatch script
cat $0

source .venv/bin/activate

num_episodes=100000
output_dir="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/procgen/coinrun_episodes"
mkdir -p $output_dir
# name="coinrun"

python generate_dataset.py --num_episodes $num_episodes --output_dir "$output_dir"

# python generate_dataset.py --env_name "$name" --num_episodes $num_episodes --output_dir "$output_dir"
# python generate_dataset.py --env_name "coinrun" --num_episodes 1000 --output_dir "/pfs/work8/workspace/ffhk/scratch/tum_ind3695-jafar_workspace/data/procgen"
[tab] slurm/common/mp4_to_npy_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=12:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_all
#SBATCH --mail-type=ALL

cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/common/preprocess_npy_to_tfrecord.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=12:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_all
#SBATCH --mail-type=ALL

cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] requirements_franz.txt (pip-requirements)
absl-py==2.2.2
annotated-types==0.7.0
astunparse==1.6.3
certifi==2025.4.26
cffi==1.17.1
charset-normalizer==3.4.2
chex==0.1.89
click==8.2.1
cloudpickle==3.1.1
dm-pix==0.4.4
docker-pycreds==0.4.0
docstring-parser==0.16
einops==0.8.1
etils==1.12.2
filelock==3.18.0
flatbuffers==25.2.10
flax==0.10.6
fsspec==2025.5.0
gast==0.6.0
gitdb==4.0.12
gitpython==3.1.44
glcontext==3.0.0
glfw==1.12.0
google-pasta==0.2.0
grpcio==1.71.0
gym==0.26.2
gym-notices==0.0.8
gym3==0.3.3
h5py==3.13.0
humanize==4.12.3
idna==3.10
imageio==2.37.0
imageio-ffmpeg==0.3.0
importlib-resources==6.5.2
jax==0.6.1
jax-cuda12-pjrt==0.6.1
jax-cuda12-plugin==0.6.1
jaxlib==0.6.1
keras==3.10.0
libclang==18.1.1
markdown==3.8
markdown-it-py==3.0.0
markupsafe==3.0.2
mdurl==0.1.2
ml-dtypes==0.5.1
moderngl==5.12.0
msgpack==1.1.0
namex==0.0.9
nest-asyncio==1.6.0
numpy==1.26.4
nvidia-cublas-cu12==12.8.4.1
nvidia-cuda-cupti-cu12==12.9.19
nvidia-cuda-nvcc-cu12==12.9.41
nvidia-cuda-runtime-cu12==12.9.37
nvidia-cudnn-cu12==9.10.1.4
nvidia-cufft-cu12==11.4.0.6
nvidia-cusolver-cu12==11.7.4.40
nvidia-cusparse-cu12==12.5.9.5
nvidia-nccl-cu12==2.26.5
nvidia-nvjitlink-cu12==12.9.41
nvidia-nvshmem-cu12==3.2.5
opt-einsum==3.4.0
optax==0.2.4
optree==0.15.0
orbax-checkpoint==0.11.13
packaging==25.0
pillow==11.2.1
platformdirs==4.3.8
procgen==0.10.7
protobuf==5.29.4
psutil==7.0.0
pycparser==2.22
pydantic==2.11.5
pydantic-core==2.33.2
pygments==2.19.1
pyyaml==6.0.2
requests==2.32.3
rich==14.0.0
scipy==1.15.3
sentry-sdk==2.29.1
setproctitle==1.3.6
setuptools==80.8.0
shtab==1.7.2
simplejson==3.20.1
six==1.17.0
smmap==5.0.2
tensorboard==2.19.0
tensorboard-data-server==0.7.2
tensorflow==2.19.0
tensorflow-io-gcs-filesystem==0.37.1
tensorstore==0.1.75
termcolor==3.1.0
toolz==1.0.0
treescope==0.1.9
typeguard==4.4.2
typing-extensions==4.13.2
typing-inspection==0.4.1
tyro==0.9.21
urllib3==2.4.0
wandb==0.19.11
werkzeug==3.1.3
wheel==0.45.1
wrapt==1.17.2
zipp==3.21.0
ffmpeg-python==0.2.0
tqdm==4.67.1
ipykernel
pre-commit
[tab] slurm/common/requirements_franz.txt (pip-requirements)
absl-py==2.2.2
annotated-types==0.7.0
astunparse==1.6.3
certifi==2025.4.26
cffi==1.17.1
charset-normalizer==3.4.2
chex==0.1.89
click==8.2.1
cloudpickle==3.1.1
dm-pix==0.4.4
docker-pycreds==0.4.0
docstring-parser==0.16
einops==0.8.1
etils==1.12.2
filelock==3.18.0
flatbuffers==25.2.10
flax==0.10.6
fsspec==2025.5.0
gast==0.6.0
gitdb==4.0.12
gitpython==3.1.44
glcontext==3.0.0
glfw==1.12.0
google-pasta==0.2.0
grpcio==1.71.0
gym==0.26.2
gym-notices==0.0.8
gym3==0.3.3
h5py==3.13.0
humanize==4.12.3
idna==3.10
imageio==2.37.0
imageio-ffmpeg==0.3.0
importlib-resources==6.5.2
jax==0.6.1
jax-cuda12-pjrt==0.6.1
jax-cuda12-plugin==0.6.1
jaxlib==0.6.1
keras==3.10.0
libclang==18.1.1
markdown==3.8
markdown-it-py==3.0.0
markupsafe==3.0.2
mdurl==0.1.2
ml-dtypes==0.5.1
moderngl==5.12.0
msgpack==1.1.0
namex==0.0.9
nest-asyncio==1.6.0
numpy==1.26.4
nvidia-cublas-cu12==12.8.4.1
nvidia-cuda-cupti-cu12==12.9.19
nvidia-cuda-nvcc-cu12==12.9.41
nvidia-cuda-runtime-cu12==12.9.37
nvidia-cudnn-cu12==9.10.1.4
nvidia-cufft-cu12==11.4.0.6
nvidia-cusolver-cu12==11.7.4.40
nvidia-cusparse-cu12==12.5.9.5
nvidia-nccl-cu12==2.26.5
nvidia-nvjitlink-cu12==12.9.41
nvidia-nvshmem-cu12==3.2.5
opt-einsum==3.4.0
optax==0.2.4
optree==0.15.0
orbax-checkpoint==0.11.13
packaging==25.0
pillow==11.2.1
platformdirs==4.3.8
procgen==0.10.7
protobuf==5.29.4
psutil==7.0.0
pycparser==2.22
pydantic==2.11.5
pydantic-core==2.33.2
pygments==2.19.1
pyyaml==6.0.2
requests==2.32.3
rich==14.0.0
scipy==1.15.3
sentry-sdk==2.29.1
setproctitle==1.3.6
setuptools==80.8.0
shtab==1.7.2
simplejson==3.20.1
six==1.17.0
smmap==5.0.2
tensorboard==2.19.0
tensorboard-data-server==0.7.2
tensorflow==2.19.0
tensorflow-io-gcs-filesystem==0.37.1
tensorstore==0.1.75
termcolor==3.1.0
toolz==1.0.0
treescope==0.1.9
typeguard==4.4.2
typing-extensions==4.13.2
typing-inspection==0.4.1
tyro==0.9.21
urllib3==2.4.0
wandb==0.19.11
werkzeug==3.1.3
wheel==0.45.1
wrapt==1.17.2
zipp==3.21.0
ffmpeg-python==0.2.0
tqdm==4.67.1
ipykernel
pre-commit
[tab] sbatch_scripts/coinrun/train_tokenizer_coinrun.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --error=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_tokenizer_coinrun
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared'

CHECKPOINT_DIR=$ws_dir/checkpoints/tokenizer/$job_name_$slurm_job_id
LOG_DIR=$ws_dir/logs/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR
mkdir -p $LOG_DIR

data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/coinrun/coinrun_tfrecords'

srun python train_tokenizer.py \
    --batch_size=192 \
    --ckpt_dir $CHECKPOINT_DIR \
    --log_checkpoint_interval=1000 \
    --log_image_interval=100 \
    --name $job_name_$slurm_job_id \
    --image_height 64 \
    --image_width 64 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[content] slurm/common/default_gpu.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --error=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_tokenizer_coinrun
#SBATCH --mail-type=ALL
[tab] slurm/common/default_gpu copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --error=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_tokenizer_coinrun
#SBATCH --mail-type=ALL
[tab] slurm/common/default_cpu.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --error=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_tokenizer_coinrun
#SBATCH --mail-type=ALL
[tab] sbatch_scripts/preprocess/mp4_to_npy_6xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_6xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try/6xx/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/6xx"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[content] slurm/common/default_cpu.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_6xx
#SBATCH --mail-type=ALL
#
[tab] sbatch_scripts/preprocess/mp4_to_npy_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=12:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_all
#SBATCH --mail-type=ALL

cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/jobs/mp4_to_npy_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=12:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_all
#SBATCH --mail-type=ALL

cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/train_lam/train_lam_full.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=16:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_lam/%x_%j.log
#SBATCH --error=logs/logs_training_lam/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_lam_ful_tfrecord
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/lam/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --log_checkpoint_interval=500 \
    --log_image_interval=50 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/jobs/train_lam_full.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=16:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_lam/%x_%j.log
#SBATCH --error=logs/logs_training_lam/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_lam_ful_tfrecord
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/lam/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --log_checkpoint_interval=500 \
    --log_image_interval=50 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] sbatch_scripts/train_tokenizer/train_tokenizer_knoms_overfit_tfrecord_full.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --error=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_tokenizer_knoms_overfit_tfrecord_full
#SBATCH --mail-type=ALL


# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/data/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR


data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --num_steps=300000 \
    --warmup_steps=10000 \
    --log_checkpoint_interval=500 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log_image_interval=100 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/jobs/train_tokenizer_knoms_overfit_tfrecord_full.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --error=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_tokenizer_knoms_overfit_tfrecord_full
#SBATCH --mail-type=ALL


# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/data/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR


data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --num_steps=300000 \
    --warmup_steps=10000 \
    --log_checkpoint_interval=500 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log_image_interval=100 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] sbatch_scripts/train_dyn/train_dyn_knoms_full.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_dyn/%x_%j.log
#SBATCH --error=logs/logs_training_dyn/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_dyn_knoms_full
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID


CHECKPOINT_DIR=$ws_dir/checkpoints/dyn/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR


data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'


lam_checkpoint='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/lam/3277318/lam_1750444032_10000'
tokenizer_checkpoint='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/tokenizer/3273828/tokenizer_1750269881_15500'

srun python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --lam_checkpoint $lam_checkpoint \
    --tokenizer_checkpoint $tokenizer_checkpoint \
    --batch_size=48 \
    --log_checkpoint_interval=500 \
    --log_image_interval=50 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/jobs/train_dyn_knoms_full.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_dyn/%x_%j.log
#SBATCH --error=logs/logs_training_dyn/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_dyn_knoms_full
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID


CHECKPOINT_DIR=$ws_dir/checkpoints/dyn/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR


data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'


lam_checkpoint='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/lam/3277318/lam_1750444032_10000'
tokenizer_checkpoint='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/tokenizer/3273828/tokenizer_1750269881_15500'

srun python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --lam_checkpoint $lam_checkpoint \
    --tokenizer_checkpoint $tokenizer_checkpoint \
    --batch_size=48 \
    --log_checkpoint_interval=500 \
    --log_image_interval=50 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/jobs/mp4_to_npy_6xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_6xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try/6xx/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/6xx"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_6xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_6xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/6xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/6xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_10xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_10xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/10xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/10xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_10xx copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_10xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/10xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/10xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_10xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/10xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/10xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/jobs/npy_to_tfrecord_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=12:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_open_ai
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord/"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 500
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/jobs/train_dyn_knoms.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_dyn/%x_%j.log
#SBATCH --error=logs/logs_training_dyn/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_dyn_knoms_full
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID


CHECKPOINT_DIR=$ws_dir/checkpoints/dyn/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR


data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'


lam_checkpoint='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/lam/3277318/lam_1750444032_10000'
tokenizer_checkpoint='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/tokenizer/3273828/tokenizer_1750269881_15500'

srun python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --lam_checkpoint $lam_checkpoint \
    --tokenizer_checkpoint $tokenizer_checkpoint \
    --batch_size=48 \
    --log_checkpoint_interval=500 \
    --log_image_interval=50 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/jobs/train_lam_knoms.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=16:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_lam/%x_%j.log
#SBATCH --error=logs/logs_training_lam/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_lam_ful_tfrecord
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/lam/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --log_checkpoint_interval=500 \
    --log_image_interval=50 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/jobs/alfred/mp4_to_npy_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=12:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_all
#SBATCH --mail-type=ALL

cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/jobs/alfred/preproc_mp4_to_npy_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=12:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_all
#SBATCH --mail-type=ALL

cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/templates/default_cpu.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/%x_%j.log
#SBATCH --error=logs/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=default_cpu
#SBATCH --mail-type=ALL
[tab] slurm/templates/default_gpu.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=1:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=1
#SBATCH --gres=gpu:1
#SBATCH --output=logs/%x_%j.log
#SBATCH --error=logs/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=default_gpu
#SBATCH --mail-type=ALL