p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] dataset_duplicates.ipynb (python)
base = "/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m"

train_dir = os.path.join(base, "train")
test_dir = os.path.join(base, "test")
val_dir = os.path.join(base, "val")

train_array_record_files = get_array_record_files(train_dir)
# test_array_record_files = get_array_record_files(test_dir)
# val_array_record_files = get_array_record_files(val_dir)

array_record_files = train_array_record_files# + test_array_record_files + val_array_record_files
print(f"Found {len(array_record_files)} files to process.")

num_processes = multiprocessing.cpu_count()
print(f"Using {num_processes} worker processes.")
[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/dev/metadata.json (json)
{"env": "coinrun", "num_actions": 0, "num_episodes_train": 1, "num_episodes_val": 1, "num_episodes_test": 1, "avg_episode_len_train": NaN, "avg_episode_len_val": NaN, "avg_episode_len_test": NaN, "episode_metadata_train": [], "episode_metadata_val": [], "episode_metadata_test": []}
[tab] slurm/jobs/mihir/horeka/preprocessing/doom_chunked_test.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=10:00:00
#SBATCH --partition=large
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/coinrun/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/coinrun/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/dev \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 1 \
    --num_episodes_val 1 \
    --num_episodes_test 1 \
    --chunk_size 100 \
    --chunks_per_file 10
[content] data/jasmine_data/ViZDoomPPO/load_model_generate_dataset.py (python)

                episode_obs_chunks.append(observations_seq[:args.chunk_size])
                episode_act_chunks.append(actions_seq[:args.chunk_size])
                observations_seq = observations_seq[args.chunk_size:]
                actions_seq = actions_seq[args.chunk_size:]
[content] data/jasmine_data/ViZDoomPPO/load_model_generate_dataset.py (python)

            if w != target_width:
                target_height = int(round(h * (target_width / float(w))))
                resample_filter = Image.LANCZOS
                img = img.resize(
                    (target_width, target_height), resample=resample_filter
                )
[tab] slurm/jobs/mihir/horeka/preprocessing/doom_chunked copy.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 50_000 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom_chunked_10m.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 50_000 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom_chunked_10m copy.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 10_000 \
[tab] slurm/jobs/mihir/horeka/preprocessing/doom_chunked_50m.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/doom/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

# export PYTHONUNBUFFERED=1


# source .venv/bin/activate

source .venv/bin/activate

python jasmine_data/ViZDoomPPO/load_model_generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_doom/doom_episodes_50m \
    --agent_path /home/hk-project-p0023960/tum_cte0515/Projects/jasmine/data/jasmine_data/ViZDoomPPO/logs/models/deathmatch_simple_bak/best_model.zip \
    --num_episodes_train 10_000 \