p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] experiments/train_dynamics_nnx.sh (shellscript)
#!/usr/bin/env bash

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

CHECKPOINT_DIR=$PWD/checkpoints/dynamics_nnx
mkdir -p $CHECKPOINT_DIR

tokenizer_ckpt_dir=$PWD/checkpoints/tokenizer_nnx

env | grep SLURM

srun python train_dynamics.py \
    --save_ckpt \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --log_checkpoint_interval=5 \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir \
[tab] experiments/sample.sh (shellscript)
#!/usr/bin/env bash

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/batchsize_scaling/dynamics_cotraining/linear-lr/train_dynamics_batch_size_scaling_linear_lr_16_node/
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python sample.py \
    --checkpoint $CHECKPOINT_DIR \
    --batch_size=1 \
    --start_frame=0 \
    --seq_len=2 \
    --data_dir $array_records_dir
[tab] experiments/train_tokenizer_nnx.sh (shellscript)
#!/usr/bin/env bash

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

CHECKPOINT_DIR=$PWD/checkpoints/tokenizer_nnx
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --save_ckpt \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --log_checkpoint_interval=5 \
    --data_dir $array_records_dir \