p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] train_lam.py (python)
                # Move checkpoint from tmp_ckpt_dir to ckpt_dir if ckpt_dir is specified
                if args.ckpt_dir:
                    src = os.path.join(os.getcwd(), args.tmp_ckpt_dir, f"tokenizer_{ts}_{step}")
                    dst = os.path.join(args.ckpt_dir, f"tokenizer_{ts}_{step}")
                    # Remove destination if it exists to avoid errors
                    if os.path.exists(dst):
                        if os.path.isdir(dst):
                            shutil.rmtree(dst)
                        else:
                            os.remove(dst)
                    shutil.move(src, dst)
[tab] scripts_horeka/modelsize_scaling/tokenizer/tester.sh (shellscript)

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name="debug"
slurm_job_id="debug-mihir"

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

XLA_FLAGS=--xla_gpu_autotune_level=0 srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=5e-5 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-74M-$slurm_job_id \
    --tags tokenizer model-size-scaling 74M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=640 \
    --num_blocks=10 \
    --num_heads=10 \
    --latent_dim=48 \
    --num_latents=1536
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_74M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_37M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=7e-5 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-74M-$slurm_job_id \
    --tags tokenizer model-size-scaling 74M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=640 \
    --num_blocks=10 \
    --num_heads=10 \
    --latent_dim=48 \
    --num_latents=1536
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_37M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_37M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-model-size-scaling-38M-$slurm_job_id \
    --tags tokenizer model-size-scaling 38M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[content] scripts_horeka/modelsize_scaling/tokenizer/tester.sh (shellscript)
python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=7e-5 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-74M-$slurm_job_id \
    --tags tokenizer model-size-scaling 74M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=640 \
    --num_blocks=10 \
    --num_heads=10 \
    --latent_dim=48 \
    --num_latents=1536
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_227M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_227M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=3e-5 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-227M-$slurm_job_id \
    --tags tokenizer model-size-scaling L2 227M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=1024 \
    --num_blocks=12 \
    --num_heads=16 \
    --latent_dim=128 \
    --num_latents=2048
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_127M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_127M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=5e-5 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-127M-$slurm_job_id \
    --tags tokenizer model-size-scaling L1 127M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=768 \
    --num_blocks=12 \
    --num_heads=12 \
    --latent_dim=64 \
    --num_latents=2048
[tab] scripts_horeka/modelsize_scaling/tokenizer/_train_tokenizer_200M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=12
#SBATCH --ntasks-per-node=4
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_200M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-200M-$slurm_job_id \
    --tags tokenizer model-size-scaling L3 200M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=1152 \
    --num_blocks=16 \
    --num_heads=16 \
    --latent_dim=128 \
    --num_latents=4096
[content] train_tokenizer.py (python)
            if step % args.log_checkpoint_interval == 0:
                ckpt = {"model": train_state}
                orbax_checkpointer = PyTreeCheckpointer()
                save_args = orbax_utils.save_args_from_target(ckpt)
                orbax_checkpointer.save(
                    os.path.join(args.tmp_ckpt_dir, f"tokenizer_{ts}_{step}"),
                    ckpt,
                    save_args=save_args,
                )
                if jax.process_index() == 0 and args.ckpt_dir:
                    # ctx = multiprocessing.get_context("spawn")
                    # move = ctx.Process(
                    #     target=lambda: shutil.move(
                    #         os.path.join(args.tmp_ckpt_dir, f"tokenizer_{ts}_{step}"),
                    #         os.path.join(args.ckpt_dir, f"tokenizer_{ts}_{step}"),
                    #     ),
                    #     daemon=True,
                    # )
                    # move.start()
                    shutil.move(
                             os.path.join(args.tmp_ckpt_dir, f"tokenizer_{ts}_{step}"),
                             os.path.join(args.ckpt_dir, f"tokenizer_{ts}_{step}"),
                         )
[content] train_tokenizer.py (python)
            if step % args.log_checkpoint_interval == 0:
                ckpt = {"model": train_state}
                orbax_checkpointer = PyTreeCheckpointer()
                save_args = orbax_utils.save_args_from_target(ckpt)
                orbax_checkpointer.save(
                    os.path.join(args.tmp_ckpt_dir, f"tokenizer_{ts}_{step}"),
                    ckpt,
                    save_args=save_args,
                )
                if jax.process_index() == 0 and args.ckpt_dir:
                    move = multiprocessing.Process(
                        target=lambda: shutil.move(
                            os.path.join(args.tmp_ckpt_dir, f"tokenizer_{ts}_{step}"),
                            os.path.join(args.ckpt_dir, f"tokenizer_{ts}_{step}"),
                        ),
                        daemon=True,
                    )
                    move.start()
[tab] scripts_horeka/modelsize_scaling/lam/tester.sh (shellscript)

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name="debug"
slurm_job_id="debug-mihir"

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

XLA_FLAGS=--xla_gpu_autotune_level=0 srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=50 \
    --log_checkpoint_interval=5 \
    --log \
    --name=lam-model-size-scaling-39M-$slurm_job_id \
    --tags lam model-size-scaling 39M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/modelsize_scaling/lam/train_lam_38M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_model_size_scaling_38M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=lam-model-size-scaling-38M-$slurm_job_id \
    --tags lam model-size-scaling 38M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[content] train_lam.py (python)
            if step % args.log_checkpoint_interval == 0:
                ckpt = {"model": train_state}
                orbax_checkpointer = PyTreeCheckpointer()
                save_args = orbax_utils.save_args_from_target(ckpt)
                orbax_checkpointer.save(
                    os.path.join(args.tmp_ckpt_dir, f"tokenizer_{ts}_{step}"),
                    ckpt,
                    save_args=save_args,
                )
                if jax.process_index() == 0 and args.ckpt_dir:
                    move = multiprocessing.Process(
                        target=lambda: shutil.move(
                            os.path.join(args.tmp_ckpt_dir, f"tokenizer_{ts}_{step}"),
                            os.path.join(args.ckpt_dir, f"tokenizer_{ts}_{step}"),
                        ),
                        daemon=True,
                    )
                    move.start()
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_6.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_model_size_scaling_38M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=lam-model-size-scaling-38M-$slurm_job_id \
    --tags lam model-size-scaling 38M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_6 copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_action_space_scaling_6

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=lam-action-space-scaling-6-$slurm_job_id \
    --tags lam action-space-scaling 6 \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_6 copy 2.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_action_space_scaling_6

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=lam-action-space-scaling-6-$slurm_job_id \
    --tags lam action-space-scaling 6 \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_6 copy 3.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_action_space_scaling_6

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=lam-action-space-scaling-6-$slurm_job_id \
    --tags lam action-space-scaling 6 \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_8.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_action_space_scaling_6

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=lam-action-space-scaling-6-$slurm_job_id \
    --tags lam action-space-scaling 6 \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_12.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_action_space_scaling_6

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=lam-action-space-scaling-6-$slurm_job_id \
    --tags lam action-space-scaling 6 \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_20.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_action_space_scaling_6

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=lam-action-space-scaling-6-$slurm_job_id \
    --tags lam action-space-scaling 6 \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/modelsize_scaling/model_sizes.md (markdown)
# Genie 1 - Model Sizes and their configs

## Tokenizer model: sizes

default: 
| Model | model_dim | num_blocks | num_heads | latent_dim | num_latents | Est. Params |
|-------|-----------|------------|-----------|------------|-------------|-------------|
| default | 512 | 8 | 8 | 32 | 1024 | ~38M |

### scaling up 
#### (not tested yet - TODO @mihir)

| Model | model_dim | num_blocks | num_heads | latent_dim | num_latents | Est. Params |
|-------|-----------|------------|-----------|------------|-------------|-------------|
| L1 | 768 | 12 | 12 | 64 | 2048 | ~127M |
| L2 | 1024 | 12 | 16 | 128 | 2048 | ~227|


### tiny models
| Model | model_dim | num_blocks | num_heads | latent_dim | num_latents | Est. Params |
|-------|-----------|------------|-----------|------------|-------------|-------------|
| S1 | 128 | 2 | 2 | 8 | 128 | ~0.6M |
| S2 | 192 | 2 | 3 | 16 | 128 | ~1.3M |
| S3 | 256 | 3 | 4 | 16 | 256 | ~3.6M |
| S4 | 320 | 4 | 5 | 24 | 256 | ~7.4M |
| S5 | 384 | 4 | 6 | 32 | 512 | ~10M |


## Latent Action model: sizes
default: 
| Model | model_dim | num_blocks | num_heads | latent_dim | num_latents | Est. Params |
|-------|-----------|------------|-----------|------------|-------------|-------------|
| default | 512 | 8 | 8 | 32 | 6 | ~39M |

### scaling up 
#### (not tested yet - TODO @mihir)

| Name | model_dim | num_blocks | num_heads | latent_dim | num_latents | Est. Params |
|--------------|-----------|------------|-----------|------------|-------------|-------------|
| XL | 1024 | 12 | 16 | 64 | 12 | ~200M |
| L | 896 | 12 | 14 | 48 | 8 | ~150M |
| M+ | 768 | 10 | 12 | 48 | 8 | ~100M |
| M | 640 | 10 | 10 | 32 | 8 | ~70M |
| Base+ | 512 | 12 | 8 | 32 | 8 | ~55M |


### tiny models
| Name | model_dim | num_blocks | num_heads | latent_dim | num_latents | Est. Params |
|--------------|-----------|------------|-----------|------------|-------------|-------------|
| XS | 128 | 2 | 2 | 8 | 4 | ~0.9M |
| S | 160 | 2 | 2 | 8 | 4 | ~1.3M |
| S+ | 192 | 3 | 3 | 8 | 4 | ~2.4M |
| M- | 256 | 4 | 4 | 16 | 6 | ~5.4M |
| M | 320 | 6 | 4 | 16 | 6 | ~12M |


## Dynamics model: sizes 

| Config | dyna_dim | dyna_num_blocks | dyna_num_heads | Approx. Params |
|--------|----------|-----------------|---------------|----------------|
| 1 | 512 | 12 | 8   | ~36M  |
| 2 | 768 | 16 | 12  | ~110M |
| 3 | 1024 | 16 | 16 | ~180M |
| 4 | 1024 | 24 | 16 | ~270M |
| 5 | 1536 | 24 | 24 | ~500M |


### tiny models
| Config | dyna_dim | dyna_num_blocks | dyna_num_heads | Approx. Params |
|--------|----------|-----------------|---------------|----------------|
| A | 128 | 2 | 4 | ~1.5M |
| B | 256 | 2 | 4 | ~3.5M |
| C | 256 | 4 | 4 | ~6M   |
| D | 384 | 4 | 6 | ~12M  |
| E | 512 | 4 | 8 | ~18M  |
[content] scripts_horeka/action_space_scaling/train_lam_actionspace_20.sbatch (shellscript)
# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM
[content] scripts_horeka/batchsize_scaling/adjusted_lr/train_tokenizer_1_nodes.sbatch (shellscript)
# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM
[content] scripts_horeka/batchsize_scaling/adjusted_lr/train_tokenizer_2_nodes.sbatch (shellscript)
# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM
[content] scripts_horeka/batchsize_scaling/adjusted_lr/train_tokenizer_4_nodes.sbatch (shellscript)
# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM
[content] train_tokenizer.py (python)
                    shutil.move(
                             os.path.join(args.tmp_ckpt_dir, f"tokenizer_{ts}_{step}"),
                             os.path.join(args.ckpt_dir, f"tokenizer_{ts}_{step}"),
                         )
[content] train_lam.py (python)
                    shutil.move(
                             os.path.join(args.tmp_ckpt_dir, f"tokenizer_{ts}_{step}"),
                             os.path.join(args.ckpt_dir, f"tokenizer_{ts}_{step}"),
                         )
[content] genie.py (python)
        """Genie-like autoregressive sampling up to `seq_len` frames. 
        We tokenize the input frames once, autoregressively sample in token-space and detokenize
        all the generations in one forward pass, like Figure 8 in the paper. 
        Note, that for interative sampling, we would require to detokenize each sampling step, 
        as the user needs to see the frame after each action.
[tab] scripts_horeka/overfit_batch_tiny/tester.sh (shellscript)
#!/usr/bin/env bash

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name="debug"
slurm_job_id="0000"

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

# Use checkpoints from tokenizer/lam overfit sample runs
tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3309662/tokenizer_1751381446_14500/
lam_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3309663/lam_1751381446_200000

python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=100 \
    --log \
    --log_checkpoint_interval=500 \
    --name=lam-tiny-overfit-batch-$slurm_job_id \
    --tags lam overfit-batch tiny \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=192 \
    --latent_dim=8 \
    --num_latents=4 \
    --patch_size=16 \
    --num_blocks=3 \
    --num_heads=3 \
    --codebook_dropout=0.0
[tab] read_tf_record.py (python)
import tensorflow as tf
import sys
import matplotlib.pyplot as plt
import numpy as np

def get_tfrecord_shape(tfrecord_path):
    raw_dataset = tf.data.TFRecordDataset(tfrecord_path)
    for raw_record in raw_dataset.take(1):
        example = tf.train.Example()
        example.ParseFromString(raw_record.numpy())
        feature = example.features.feature
        height = feature['height'].int64_list.value[0]
        width = feature['width'].int64_list.value[0]
        channels = feature['channels'].int64_list.value[0]
        sequence_length = feature['sequence_length'].int64_list.value[0]
        print(f"Shape: ({sequence_length}, {height}, {width}, {channels})")
        break

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: python read_tf_record.py <tfrecord_file_path>")
    else:
        tfrecord_file_path = sys.argv[1]
        get_tfrecord_shape(tfrecord_file_path)

        def visualize_first_frame(tfrecord_path):
            raw_dataset = tf.data.TFRecordDataset(tfrecord_path)
            for raw_record in raw_dataset.take(1):
                example = tf.train.Example()
                example.ParseFromString(raw_record.numpy())
                feature = example.features.feature
                raw_video = feature['raw_video'].bytes_list.value[0]
                height = feature['height'].int64_list.value[0]
                width = feature['width'].int64_list.value[0]
                channels = feature['channels'].int64_list.value[0]
                sequence_length = feature['sequence_length'].int64_list.value[0]

                # Decode the raw video bytes
                video_array = np.frombuffer(raw_video, dtype=np.uint8)
                video_array = video_array.reshape((sequence_length, height, width, channels))

                # Extract the first frame
                first_frame = video_array[0]

                # Save the first frame as an image
                plt.imshow(first_frame)
                plt.axis('off')
                plt.savefig('frame.png', bbox_inches='tight', pad_inches=0)
                plt.close()
                break

        visualize_first_frame(tfrecord_file_path)
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_6.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_action_space_scaling_6

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tmp_ckpt_dir $ODFSDIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=1000 \
    --log \
    --num_latents=6 \
    --name=lam-action-space-scaling-6-$slurm_job_id \
    --tags lam action-space-scaling 6 \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_8.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_action_space_scaling_8

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tmp_ckpt_dir $ODFSDIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=1000 \
    --log \
    --num_latents=8 \
    --name=lam-action-space-scaling-8-$slurm_job_id \
    --tags lam action-space-scaling 8 \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_12.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_action_space_scaling_12

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tmp_ckpt_dir $ODFSDIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=1000 \
    --log \
    --num_latents=12 \
    --name=lam-action-space-scaling-12-$slurm_job_id \
    --tags lam action-space-scaling 12 \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/action_space_scaling/train_lam_actionspace_20.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_action_space_scaling_20

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tmp_ckpt_dir $ODFSDIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=1000 \
    --log \
    --num_latents=20 \
    --name=lam-action-space-scaling-20-$slurm_job_id \
    --tags lam action-space-scaling 20 \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/batchsize_scaling/adjusted_lr/train_tokenizer_1_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_1_node

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tmp_ckpt_dir $ODFSDIR \
    --batch_size=48 \
    --min_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=tokenizer-batch-size-scaling-1-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 1-node \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/batchsize_scaling/adjusted_lr/train_tokenizer_2_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_2_node

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tmp_ckpt_dir $ODFSDIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=1.4e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=tokenizer-batch-size-scaling-2-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 2-node \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/batchsize_scaling/adjusted_lr/train_tokenizer_4_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_4_node

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tmp_ckpt_dir $ODFSDIR \
    --batch_size=192 \
    --min_lr=0 \
    --max_lr=2.00e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=tokenizer-batch-size-scaling-4-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 4-node \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_37M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_37M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tmp_ckpt_dir $ODFSDIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=tokenizer-model-size-scaling-38M-$slurm_job_id \
    --tags tokenizer model-size-scaling 38M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_74M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_74M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tmp_ckpt_dir $ODFSDIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=7e-5 \
    --log_image_interval=500 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=tokenizer-modelsize-74M-$slurm_job_id \
    --tags tokenizer model-size-scaling 74M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=640 \
    --num_blocks=10 \
    --num_heads=10 \
    --latent_dim=48 \
    --num_latents=1536
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_127M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_127M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

ODFSDIR=/mnt/odfs/$SLURM_JOB_ID/stripe_default/checkpoints/
mkdir -p $ODFSDIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --tmp_ckpt_dir $ODFSDIR \
    --batch_size=96 \
    --min_lr=0 \
    --max_lr=5e-5 \
    --log_image_interval=500 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=tokenizer-modelsize-127M-$slurm_job_id \
    --tags tokenizer model-size-scaling L1 127M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=768 \
    --num_blocks=12 \
    --num_heads=12 \
    --latent_dim=64 \
    --num_latents=2048