p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] jasmine/baselines/maskgit/train_dynamics_maskgit.py (python)
        restored = checkpoint_manager.restore(restore_step, args=restore_args)
        restored_optimizer_state = restored["model_state"]
        nnx.update(optimizer, restored_optimizer_state)
        train_iterator = restored["train_dataloader_state"]
        if val_iterator:
            val_iterator = restored["val_dataloader_state"]
        step = restore_step or 0
[tab] slurm/jobs/franz/berlin/atari/data_generation/alien.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=64
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/atari/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/atari/%x_%j.log
#SBATCH --job-name=generate_atari_alien_no_sticky_actions_data

cd data/
uv run python jasmine_data/atari/generate_atari_dataset.py \
 --output_dir=/fast/project/HFMI_SynergyUnit/jafar_ws/data/atari/alien \
 --gin_files=jasmine_data/atari/rainbow_no_sticky_actions.gin \
 --gin_bindings="atari_lib.create_atari_environment.game_name='Alien'"
[tab] data/pyproject.toml (plaintext)
[project]
name = "jasmine-data"
version = "0.1.0"
authors = [
  {name = "Alfred Nguyen", email = "alfrednguyen02@gmail.com"},
  {name = "Mihir Mahajan", email = "mihir@pdoom.org"},
  {name = "Franz Srambical", email = "franz@pdoom.org"},
]
requires-python = "==3.10.*"
dependencies = [
  "procgen>=0.10.7",
  "array-record>=0.7.2",
  "numpy",
  "hf-transfer==0.1.9",
  "huggingface-hub[cli]>=0.34.3",
  "ffmpeg-python==0.2.0",
  "tqdm>=4.67.1",
  "tyro>=0.8.5",
  "dopamine-rl",
]

[tool.uv.sources]
dopamine-rl = { path = "../third_party/dopamine", editable = true }

[build-system]
requires = ["uv_build>=0.8.22,<0.9.0"]
build-backend = "uv_build"

[tool.uv.build-backend]
module-root = ""
[tab] slurm/jobs/franz/berlin/atari/data_generation/breakout.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=64
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/atari/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/atari/%x_%j.log
#SBATCH --job-name=generate_atari_breakout_no_sticky_actions_data

cd data/
uv run python jasmine_data/atari/generate_atari_dataset.py \
 --output_dir=/fast/project/HFMI_SynergyUnit/jafar_ws/data/atari/breakout \
 --gin_files=jasmine_data/atari/rainbow_no_sticky_actions.gin \
 --gin_bindings="atari_lib.create_atari_environment.game_name='Breakout'"
[content] slurm/jobs/franz/berlin/atari/data_upload/upload_to_hf.sh (shellscript)

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/atari/data_upload/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/atari/data_upload/%x_%j.log
#SBATCH --job-name=upload_to_hf

source .venv/bin/activate
[content] slurm/jobs/franz/berlin/atari/data_upload/upload_to_hf.sh (shellscript)

python slurm/utils/mihir/upload_hf_dataset.py /fast/project/HFMI_SynergyUnit/jafar_ws/data/atari/bank_heist p-doom/atari-bank_heist --repo-type dataset
python slurm/utils/mihir/upload_hf_dataset.py /fast/project/HFMI_SynergyUnit/jafar_ws/data/atari/battle_zone p-doom/atari-battle_zone --repo-type dataset
python slurm/utils/mihir/upload_hf_dataset.py /fast/project/HFMI_SynergyUnit/jafar_ws/data/atari/boxing p-doom/atari-boxing --repo-type dataset
python slurm/utils/mihir/upload_hf_dataset.py /fast/project/HFMI_SynergyUnit/jafar_ws/data/atari/breakout p-doom/atari-breakout --repo-type dataset
[content] slurm/jobs/franz/berlin/atari/data_upload/upload_to_hf.sh (shellscript)
 slurm/utils/mihir/upload_hf_dataset.py /fast/project/HFMI_SynergyUnit/jafar_ws/data/atari/demon_attack p-doom/atari-demon_attack --repo-type dataset
python slurm/utils/mihir/upload_hf_dataset.py /fast/project/HFMI_SynergyUnit/jafar_ws/data/atari/pong p-doom/atari-pong