p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] sbatch_scripts/preprocess/mp4_to_npy_6xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_first_try_6xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try/6xx/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/6xx"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/mp4_to_npy_7xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_first_try_7xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try/7xx/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/7xx"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/mp4_to_npy_8xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_first_try_8xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try/8xx/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/8xx"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/mp4_to_npy_9xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_first_try_9xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try/9xx/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/9xx"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/mp4_to_npy_10xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_first_try_10xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try/10xx/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/10xx"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] utils/preprocess.py (python)
import dm_pix as pix
import einops
import jax
import jax.numpy as jnp


def patchify(videos: jax.Array, size: int) -> jax.Array:
    B, T, H, W, C = videos.shape
    x = jnp.pad(videos, ((0, 0), (0, 0), (0, -H % size), (0, -W % size), (0, 0)))
    return einops.rearrange(
        x, "b t (hn hp) (wn wp) c -> b t (hn wn) (hp wp c)", hp=size, wp=size
    )


def unpatchify(patches: jax.Array, size: int, h_out: int, w_out: int) -> jax.Array:
    h_pad = -h_out % size
    hn = (h_out + h_pad) // size
    x = einops.rearrange(
        patches,
        "b t (hn wn) (hp wp c) -> b t (hn hp) (wn wp) c",
        hp=size,
        wp=size,
        hn=hn,
    )
    return x[:, :, :h_out, :w_out]
[tab] sbatch_scripts/preprocess/mp4_to_npy_6xx copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_6xx_Jun_29
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/all_6xx_Jun_29/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/all_6xx_Jun_29"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_6xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_6xx_Jun_29
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/all_6xx_Jun_29/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/all_6xx_Jun_29"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/mp4_to_npy_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_6xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/6xx/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/6xx"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/preprocess_video_to_npy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:30:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-pai00039
#SBATCH --cpus-per-task=72
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=preprocess_video_to_npy
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

source .venv/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_npy_tmp"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[terminal_command] TERMINAL ()
python utils/preprocess_video_to_npy.py --input_path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/6xx/10fps_160x90 --output_path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/6xx/10fps_160x90
[terminal_command] TERMINAL ()
python utils/preprocess_video_to_npy.py --input_path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/6xx/10fps_160x90 --output_path /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/6xx/10^C
[terminal_command] TERMINAL ()
python utils/preprocess_dataset.py --source_data_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/6xx/10fps_160x90 --output_tfrecords_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/6xx/10fps_160x90 --num_shards 100
[terminal_command] TERMINAL ()
kpython utils/preprocess_dataset.py --source_data_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/6xx/10fps_160x90  --output_tfrecords_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/6xx/10fps_160x90 --num_shards 100^C
[terminal_command] TERMINAL ()
python utils/preprocess_dataset.py --source_data_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/6xx/10fps_160x90  --output_tfrecords_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/6xx/10fps_160x90 --num_shards 100
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_6xx copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=00:10:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_6xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/6xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/6xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_7xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=00:10:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_6xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/6xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/6xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_7xx copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_7xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/7xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/7xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_8xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_7xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/7xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/7xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_8xx copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_8xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/8xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/8xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_9xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_8xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/8xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/8xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_9xx copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_9xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/9xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/9xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] sbatch_scripts/preprocess/npy_to_tfrecord_10xx.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_9xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/9xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/9xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] logs/logs_preprocessing/mp4_to_npy_openai_all_3290234.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=00:10:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_all
#SBATCH --mail-type=ALL

cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"


/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy//10fps_160x90
Number of processes: 152
Converting mp4 to npy files...
slurmstepd: error: *** JOB 3290234 ON hkn0364 CANCELLED AT 2025-06-25T10:57:43 DUE TO TIME LIMIT ***

============================= JOB FEEDBACK =============================

Job ID: 3290234
Cluster: hk
User/Group: tum_ind3695/hk-project-pai00039
Account: hk-project-p0023960
State: TIMEOUT (exit code 0)
Partition: cpuonly
Nodes: 1
Cores per node: 152
Nodelist: hkn0364
CPU Utilized: 01:05:48
CPU Efficiency: 4.29% of 1-01:35:12 core-walltime
Job Wall-clock time: 00:10:06
Starttime: Wed Jun 25 10:47:37 2025
Endtime: Wed Jun 25 10:57:43 2025
Memory Utilized: 26.06 GB
Memory Efficiency: 11.15% of 233.79 GB
Energy Consumed: 381371 Joule / 105.936388888889 Watthours
Average node power draw: 629.325082508251 Watt
[content] logs/logs_preprocessing/npy_to_tfrecord_6xx_3290230.log (log)
INFO:root:  Processed 1000/26960 episodes...
INFO:root:  Processed 1100/26960 episodes...
INFO:root:  Processed 1200/26960 episodes...
INFO:root:  Processed 1300/26960 episodes...
INFO:root:  Processed 1400/26960 episodes...
INFO:root:  Processed 1500/26960 episodes...
INFO:root:  Processed 1600/26960 episodes...
INFO:root:  Processed 1700/26960 episodes...
INFO:root:  Processed 1800/26960 episodes...
INFO:root:  Processed 1900/26960 episodes...
INFO:root:  Processed 2000/26960 episodes...
INFO:root:  Processed 2100/26960 episodes...
INFO:root:  Processed 2200/26960 episodes...
INFO:root:  Processed 2300/26960 episodes...
INFO:root:  Processed 2400/26960 episodes...
INFO:root:  Processed 2500/26960 episodes...
INFO:root:  Processed 2600/26960 episodes...
INFO:root:  Processed 2700/26960 episodes...
INFO:root:  Processed 2800/26960 episodes...
INFO:root:  Processed 2900/26960 episodes...
INFO:root:  Processed 3000/26960 episodes...
INFO:root:  Processed 3100/26960 episodes...
INFO:root:  Processed 3200/26960 episodes...
INFO:root:  Processed 3300/26960 episodes...
INFO:root:  Processed 3400/26960 episodes...
INFO:root:  Processed 3500/26960 episodes...
INFO:root:  Processed 3600/26960 episodes...
INFO:root:  Processed 3700/26960 episodes...
INFO:root:  Processed 3800/26960 episodes...
INFO:root:  Processed 3900/26960 episodes...
INFO:root:  Processed 4000/26960 episodes...
slurmstepd: error: *** JOB 3290230 ON hkn1023 CANCELLED AT 2025-06-25T10:55:43 DUE TO TIME LIMIT ***

============================= JOB FEEDBACK =============================

Job ID: 3290230
Cluster: hk
User/Group: tum_ind3695/hk-project-pai00039
Account: hk-project-p0023960
State: TIMEOUT (exit code 0)
Partition: cpuonly
Nodes: 1
Cores per node: 152
Nodelist: hkn1023
CPU Utilized: 00:06:48
CPU Efficiency: 0.44% of 1-01:35:12 core-walltime
Job Wall-clock time: 00:10:06
Starttime: Wed Jun 25 10:45:37 2025
Endtime: Wed Jun 25 10:55:43 2025
Memory Utilized: 1.14 GB
Memory Efficiency: 0.49% of 233.79 GB
Energy Consumed: 161940 Joule / 44.9833333333333 Watthours
Average node power draw: 267.227722772277 Watt
[tab] utils/preprocess_video_to_npy.py (python)
import ffmpeg
import numpy as np
import os
import tyro
import multiprocessing as mp
from dataclasses import dataclass


@dataclass
class Args:
    target_width, target_height = 160, 90
    target_fps = 10
    input_path: str = (
        "/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms/"
    )
    output_path: str = (
        "/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_npy"
    )


def preprocess_video(
    idx, in_filename, output_path, target_width, target_height, target_fps
):
    print(f"Processing video {idx}")

    out, _ = (
        ffmpeg.input(in_filename)
        .filter("fps", fps=target_fps, round="up")
        .filter("scale", target_width, target_height)
        .output("pipe:", format="rawvideo", pix_fmt="rgb24")
        .run(capture_stdout=True, quiet=True)
    )

    frame_size = target_height * target_width * 3
    n_frames = len(out) // frame_size

    frames = np.frombuffer(out, np.uint8).reshape(
        n_frames, target_height, target_width, 3
    )

    output_file = f'{output_path}/{in_filename.split("/")[-1].split(".")[0]}.npy'
    if not os.path.exists(os.path.dirname(output_file)):
        os.makedirs(os.path.dirname(output_file))

    np.save(output_file, frames)
    print(f"Saved {n_frames} frames to {output_file} with shape {frames.shape}")


def get_meta_data(filename, directory):
    filepath = os.path.join(directory, filename)
    arr = np.load(filepath, mmap_mode="r")
    return filepath, arr.shape[0]


def main():
    args = tyro.cli(Args)

    output_path = f"{args.output_path}/{args.target_fps}fps_{args.target_width}x{args.target_height}"
    print(output_path)

    print(f"Number of processes: {mp.cpu_count()}")
    print("Converting mp4 to npy files...")
    pool_args = [
        (
            idx,
            args.input_path + in_filename,
            output_path,
            args.target_width,
            args.target_height,
            args.target_fps,
        )
        for idx, in_filename in enumerate(os.listdir(args.input_path))
        if in_filename.endswith(".mp4") or in_filename.endswith(".webm")
    ]

    with mp.Pool(processes=mp.cpu_count()) as pool:
        pool.starmap(preprocess_video, pool_args)
    print("Done converting mp4 to npy files")

    print("Creating metadata file...")
    metadata = []
    filenames = [
        filename
        for filename in os.listdir(output_path)
        if filename.endswith(".npy") and filename != "metadata.npy"
    ]
    pool_args = [(filename, output_path) for filename in filenames]

    with mp.Pool(processes=mp.cpu_count()) as pool:
        results = list(pool.starmap(get_meta_data, pool_args))
        metadata = [{"path": path, "length": length} for path, length in results]
    np.save(output_path + "/metadata.npy", metadata)
    print(f"Saved {len(metadata)} videos to {output_path}")


if __name__ == "__main__":
    main()
[tab] slurm/jobs/preprocess/mp4_to_npy_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=12:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_all
#SBATCH --mail-type=ALL

cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/dev/ali/preprocess/npy_to_tfrecord_10xx copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_10xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/10xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/10xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/dev/ali/preprocess/npy_to_tfrecord__open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_10xx
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_npy/10xx/10fps_160x90"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_first_try_tfrecord/10xx"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/dev/ali/preprocess/mp4_to_npy_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=12:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=mp4_to_npy_openai_all
#SBATCH --mail-type=ALL

cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/jobs/preprocess/npy_to_tfrecord__open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_open_ai_all
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord/"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/jobs/preprocess/npy_to_tfrecord_open_ai.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=03:00:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-p0023960
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=npy_to_tfrecord_open_ai_all
#SBATCH --mail-type=ALL
#!/usr/bin/env bash
# Log the sbatch script
cat $0

source .venv_jafar/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_npy/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord/"

start_time=$(date +%s)  
python utils/preprocess_dataset.py --source_data_dir $input_path --output_tfrecords_dir $output_path --num_shards 100
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"
[tab] slurm/jobs/train/train_dyn_knoms_full.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_dyn/%x_%j.log
#SBATCH --error=logs/logs_training_dyn/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_dyn_knoms_full
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID


CHECKPOINT_DIR=$ws_dir/checkpoints/dyn/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR


data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'


lam_checkpoint='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/lam/3277318/lam_1750444032_10000'
tokenizer_checkpoint='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/tokenizer/3273828/tokenizer_1750269881_15500'

srun python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --lam_checkpoint $lam_checkpoint \
    --tokenizer_checkpoint $tokenizer_checkpoint \
    --batch_size=48 \
    --log_checkpoint_interval=500 \
    --log_image_interval=50 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/jobs/train/train_lam_full.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=16:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_lam/%x_%j.log
#SBATCH --error=logs/logs_training_lam/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_lam_ful_tfrecord
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/lam/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --log_checkpoint_interval=500 \
    --log_image_interval=50 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/jobs/train/train_tokenizer_knoms_overfit_tfrecord_full.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --error=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_tokenizer_knoms_overfit_tfrecord_full
#SBATCH --mail-type=ALL


# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/data/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR


data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --num_steps=300000 \
    --warmup_steps=10000 \
    --log_checkpoint_interval=500 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log_image_interval=100 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/jobs/train/train_tokenizer_full.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=24:00:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --error=logs/logs_training_tokenizer/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=train_tokenizer_knoms_overfit_tfrecord_full
#SBATCH --mail-type=ALL


# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/data/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR


data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --num_steps=300000 \
    --warmup_steps=10000 \
    --log_checkpoint_interval=500 \
    --seed=0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --log_image_interval=100 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/jobs/sample/sample_knoms.sh (shellscript)

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4

source .venv_jafar/bin/activate

data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'
checkpoint_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/dyn/3282489/genie_1750610596_12500'
seq_len=16
output_dir='sample_results/knoms/'

python sample.py --checkpoint $checkpoint_dir --data_dir $data_dir --batch_size 12 --seq_len $seq_len --output_dir $output_dir
# CUDA_VISIBLE_DEVICES=0 python sample.py --checkpoint $checkpoint_dir --data_dir $data_dir --batch_size 12