p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/jobs/mihir/horeka/doom/resolution80x60/train_tokenizer_default_1gpu.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_default_single_gpu

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m/val


job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python jasmine/train_tokenizer.py \
    --save_ckpt \
    --image_height=240 \
    --image_width=320 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=8 \
    --init_lr=0 \
    --log \
    --name=doom-tokenizer-default-$slurm_job_id \
    --tags tokenizer doom default \
    --entity instant-uv \
    --project jafar \
    --data_dir $array_records_dir_train \
    --val_data_dir $array_records_dir_val
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_10m_60x80.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_10m_60x80

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_10m_60x80_fixed \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 10_000 \
    --target_width 80 \
    --target_height 60 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_10m.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_10m

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_10m \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 10_000 \
[tab] data/pyproject.toml (plaintext)
[project]
name = "jasmine-data"
version = "0.1.0"
authors = [
  {name = "Alfred Nguyen", email = "alfrednguyen02@gmail.com"},
  {name = "Mihir Mahajan", email = "mihir@pdoom.org"},
  {name = "Franz Srambical", email = "franz@pdoom.org"},
]
requires-python = "==3.10.*"
dependencies = [
  "procgen>=0.10.7",
  "gym3>=0.3.3",
  "array-record>=0.7.2",
  "numpy",
  "hf-transfer==0.1.9",
  "huggingface-hub[cli]>=0.34.3",
  "ffmpeg-python==0.2.0",
  "pillow>=11.3.0",
  "tqdm>=4.67.1",
  "tyro>=0.8.5",
<<<<<<< HEAD
  "vizdoom",
=======
  "gymnasium[atari, accept-rom-license]==0.29.1",
  "minatar>=1.0.15",
  "torch>=2.8.0",
  "tensorboard>=2.20.0",
  "opencv-python>=4.6.0.66,<5",
  "einops>=0.8.0",
>>>>>>> prepend-action-maskgit
]

[build-system]
requires = ["uv_build>=0.8.22,<0.9.0"]
build-backend = "uv_build"

[tool.uv.build-backend]
module-root = ""
[tab] slurm/jobs/mihir/horeka/doom/train_tokenizer_default_1node.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_default_single_node

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m/val


job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python jasmine/train_tokenizer.py \
    --save_ckpt \
    --image_height=240 \
    --image_width=320 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=32 \
    --init_lr=0 \
    --log \
    --name=doom-tokenizer-default-1node-$slurm_job_id \
    --tags tokenizer doom default \
    --entity instant-uv \
    --project jafar \
    --data_dir $array_records_dir_train \
    --val_data_dir $array_records_dir_val
[tab] slurm/jobs/mihir/horeka/doom/resolution120x160/train_tokenizer_default_1node.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_default_single_node

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m/val


job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python jasmine/train_tokenizer.py \
    --save_ckpt \
    --image_height=240 \
    --image_width=320 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=32 \
    --init_lr=0 \
    --log \
    --name=doom-tokenizer-default-1node-$slurm_job_id \
    --tags tokenizer doom default \
    --entity instant-uv \
    --project jafar \
    --data_dir $array_records_dir_train \
    --val_data_dir $array_records_dir_val
[tab] slurm/jobs/mihir/horeka/doom/resolution60x80/train_tokenizer_default_1gpu.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_default_single_gpu_60x80

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_10m_60x80_fixed/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_10m_60x80_fixed/val


job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python jasmine/train_tokenizer.py \
    --save_ckpt \
    --image_height=60 \
    --image_width=80 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --init_lr=0 \
    --log \
    --name=doom-tokenizer-default-60x80-$slurm_job_id \
    --tags tokenizer doom default 60x80 \
    --entity instant-uv \
    --project jafar \
    --data_dir $array_records_dir_train \
    --val_data_dir $array_records_dir_val