p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] train_tokenizer.py (python)

    handler_registry.add(
        "train_dataloader_state",
        grain.checkpoint.CheckpointSave,
        cast(ocp.handlers.CheckpointHandler, grain.checkpoint.CheckpointHandler),
    )
    handler_registry.add(
        "train_dataloader_state",
        grain.checkpoint.CheckpointRestore,
        cast(ocp.handlers.CheckpointHandler, grain.checkpoint.CheckpointHandler),
    )
[tab] slurm/jobs/alfred/helmholtz_cluster/jafar_og_reproduction/og_coinrun_lam_reproduction.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=3-00:00:00
#SBATCH --cpus-per-task=8
#SBATCH --partition=gpu_p
#SBATCH --reservation=haicu_stefan
#SBATCH --qos=gpu_long
#SBATCH --gres=gpu:1
#SBATCH --output=/lustre/groups/haicu/workspace/alfred.nguyen/jafar_worksapce/logs/coinrun/lam/%x_%j.log
#SBATCH --error=/lustre/groups/haicu/workspace/alfred.nguyen/jafar_worksapce/logs/coinrun/lam/%x_%j.log
#SBATCH --job-name=train_lam_coinrun_og_reproduction

# Log the sbatch script
cat $0

source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

tags="coinrun_og reproduction 10k_dataset lam helmholtz_reproduction"

npy_records_dir="/lustre/groups/haicu/workspace/alfred.nguyen/jafar_worksapce/data/coinrun_episodes/coinrun_episodes"

CHECKPOINT_DIR="/lustre/groups/haicu/workspace/alfred.nguyen/jafar_worksapce/checkpoints/coinrun/lam/${job_name}_${slurm_job_id}"
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --log_image_interval=1000 \
    --log \
    --name="${job_name}_${slurm_job_id}" \
    --tags $tags \
    --entity instant-uv \
    --project jafar \
    --data_dir $npy_records_dir
[tab] slurm/jobs/mihir/horeka/jafar_og_reproduction/generate_dataset_10m_gt_actions_train_val_test.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=12:00:00
#SBATCH --partition=large
#SBATCH --cpus-per-task=8
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/data_coinrun/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/data_coinrun/%x_%j.log
#SBATCH --job-name=generate_coinrun_dataset_10m_gt_actions


# Log the sbatch script
cat $0
source .venv/bin/activate

python generate_dataset.py \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split_2 \
    --min_episode_length 1000
[tab] slurm/dev/mihir/horeka/generate_dataset_10m.sh (shellscript)
#!/usr/bin/env bash

python generate_dataset.py \
    --num_episodes_train 100 \
    --num_episodes_val 50 \
    --num_episodes_test 50 \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/dev \
    --min_episode_length 1000