p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/jobs/mihir/horeka/doom/train_tokenizer_default_1gpu.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/doom/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_default_single_gpu

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m/val


job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python jasmine/train_tokenizer.py \
    --save_ckpt \
    --image_height=240 \
    --image_width=320 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=8 \
    --init_lr=0 \
    --log \
    --name=doom-tokenizer-default-$slurm_job_id \
    --tags tokenizer doom default \
    --entity instant-uv \
    --project jafar \
    --data_dir $array_records_dir_train \
    --val_data_dir $array_records_dir_val
[terminal_command] TERMINAL ()
[tum_cte0515@hkn1991 jasmine]$ idling[tum_cte0515@hkn1991 jasmine]$ queue[tum_cte0515@hkn1991 jasmine]$ cd $ws_dir[tum_cte0515@hkn1991 tum_ind3695-jafa_ws_shared]$ queue[tum_cte0515@hkn1991 tum_ind3695-jafa_ws_shared]$ fqueue
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_1m_low_res.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m_low_res \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 1_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_10m_low_res.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_10m_low_res \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 10_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_1m.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 1_000 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_10m.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_10m \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 10_000 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_50m.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=42:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 50_000
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_50m_low_res.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=42:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m_low_res \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 50_000 \
    --target_width 160 \
    --target_height 120 \
[tab] data/jasmine_data/ViZDoomPPO/explore_dataset.py (python)
import matplotlib.pyplot as plt
import base64
import io
from PIL import Image
import numpy as np
from matplotlib.animation import FuncAnimation
import pyarrow.parquet as pq


# Example script to open the parquet file and get the data
parquet_path = "./concatenated.parquet"
table = pq.read_table(parquet_path)
# Convert to pandas DataFrame
df = table.to_pandas()

# Sort by step_id
df_sorted = df.sort_values("step_id")

# Get the first row
first_row = df_sorted.iloc[27]


# Function to decode base64 image
def decode_image(b64_string):
    image_data = base64.b64decode(b64_string)
    return np.array(Image.open(io.BytesIO(image_data)))


# Decode all images in the first row
images = [decode_image(img) for img in first_row["images"]]

# Create figure and axis
fig, ax = plt.subplots(figsize=(10, 8))
plt.axis("off")


# Animation function
def animate(i):
    ax.clear()
    ax.imshow(images[i])
    ax.set_title(
        f"Episode {first_row['episode_id']}, Step {first_row['step_id']}, Frame {i+1}/{len(images)}"
    )
    ax.axis("off")


# Create animation
anim = FuncAnimation(fig, animate, frames=len(images), interval=200, repeat=True)

plt.show()

# Print other information about the first row
print(f"Episode ID: {first_row['episode_id']}")
print(f"Step ID: {first_row['step_id']}")
print(f"Health: {first_row['health']}")
print(f"Actions: {first_row['actions']}")
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_test.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=10:00:00
#SBATCH --partition=large
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/coinrun/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/coinrun/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/dev \
    --agent_path $(pwd)/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 100 \
    --num_episodes_val 100 \
    --num_episodes_test 100 \
    --chunk_size 100 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_1m_low_res copy.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_1m_low_res

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m_low_res_fixed \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 1_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_1m_60x80.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_1m_low_res

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m_low_res_fixed \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 1_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_1m_120x160.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_1m_low_res

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_1m_low_res_fixed \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 1_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_10m_low_res copy.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_10m_low_res

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_10m_low_res_fixed \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 10_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_10m_60x80.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_10m_low_res

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_10m_low_res_fixed \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 10_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_50m_low_res copy.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=42:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_50m_low_res

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m_low_res_fixed \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 50_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_50m_60x80.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=42:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_50m_low_res

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m_low_res_fixed \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 50_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_50m_120x160.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=42:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_50m_120x160

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m_120x160_fixed \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 50_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_10m_120x160.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_doom_10m_120x160

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_10m_120x160_fixed \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 10_000 \
    --target_width 160 \
    --target_height 120 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked_test._sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=10:00:00
#SBATCH --partition=large
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/coinrun/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/coinrun/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/dev \
    --agent_path $(pwd)/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 100 \
    --num_parallel_envs 100 \
    --target_width 80 \
    --target_height 60
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 50_000 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom/doom_chunked._sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 50_000 \
[tab] dataset_duplicates.ipynb (python)

print(f"
Found {len(duplicates)} sets of duplicate videos.")
if duplicates:
    # Print the first 5 duplicate sets as an example
    for i, (h, paths) in enumerate(duplicates.items()):
        print(f"  - Hash: {h[:10]}... ({len(paths)} files)")
        for path in paths:
            print(f"    - {os.path.basename(path)}")
[tab] dataset_duplicates.ipynb (python)
base = "/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m"

train_dir = os.path.join(base, "train")
test_dir = os.path.join(base, "test")
val_dir = os.path.join(base, "val")

train_array_record_files = get_array_record_files(train_dir)
test_array_record_files = get_array_record_files(test_dir)
val_array_record_files = get_array_record_files(val_dir)

array_record_files = train_array_record_files + test_array_record_files + val_array_record_files
print(f"Found {len(array_record_files)} files to process.")

num_processes = multiprocessing.cpu_count()
print(f"Using {num_processes} worker processes.")
[tab] dataset_duplicates.ipynb (python)
# Find episode level duplicates
duplicate_episode = defaultdict(list)

# The 'with' statement ensures the pool is properly closed
with multiprocessing.Pool(processes=num_processes) as pool:
    # Use pool.imap_unordered for efficiency. It processes items as they are submitted
    # and returns results as they complete, which is perfect for progress bars.
    # We wrap the result iterator with tqdm to show progress.
    results = pool.imap_unordered(get_episode_level_hash, array_record_files)
    
    print("Processing files and calculating hashes...")
    for result in tqdm(results, total=len(array_record_files)):
        if result:  # Ensure the worker didn't return None due to an error
            video_hash, video_path = result
            duplicate_episode[video_hash].append(video_path)