p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] scripts_horeka/overfit_batch_tiny/train_tokenizer_overfit_batch.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --mail-user=mihir.mahajan2002@gmail.com
#SBATCH --job-name=train_tokenizer_minecraft_overfit_batch
#SBATCH --mem=50G
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log \
    --log_checkpoint_interval=500 \
    --name=tokenizer-tiny-overfit-batch-$slurm_job_id \
    --tags tokenizer overfit-batch tiny \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim 256 \
    --latent_dim 16 \
    --num_latents 256 \
    --patch_size 4 \
    --num_blocks 3 \
    --num_heads 4 \
    --codebook_dropout 0.0
[tab] scripts_horeka/train_tokenizer.sh (shellscript)
#!/usr/bin/env bash

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name="debug"
slurm_job_id="0000"

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=4.24e-4 \
    --max_lr=4.24e-4 \
    --log_image_interval=100 \
    --log \
    --name=test-tokenizer-tags-$slurm_job_id \
    --tags test tokenizer debug \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=128 \
    --num_blocks=2 \
    --num_heads=2 \
    --latent_dim=8 \
    --num_latents=128
[content] scripts_horeka/batchsize_scaling/adjusted_lr/tester.sh (shellscript)
#!/usr/bin/env bash

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name="debug"
slurm_job_id="0000"

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=4.24e-4 \
    --max_lr=4.24e-4 \
    --log_image_interval=100 \
    --log \
    --name=test-tokenizer-tags-$slurm_job_id \
    --tags test tokenizer debug \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=128 \
    --num_blocks=2 \
    --num_heads=2 \
    --latent_dim=8 \
    --num_latents=128
[content] scripts_horeka/batchsize_scaling/adjusted_lr/tester.sh (shellscript)
srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --min_lr=3e-4 \
    --max_lr=3e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-1-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 1-node time-step \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[content] utils/visualizer.py (python)

            # INSERT_YOUR_CODE
            def visualize_and_save_batch(batch, filename="batch_visualization.png", max_samples=8):
                """
                Visualizes a batch of videos (or images) and saves as a PNG.
                Args:
                    batch: jax/numpy array of shape (B, T, H, W, C) or (B, H, W, C)
                    filename: output filename
                    max_samples: maximum number of batch elements to visualize
                """
                import matplotlib.pyplot as plt
                import numpy as np

                arr = np.asarray(batch)
                if arr.ndim == 5:
                    # (B, T, H, W, C): show first frame of each sample
                    arr = arr[:, 0]  # (B, H, W, C)
                arr = arr[:max_samples]
                B = arr.shape[0]
                fig, axes = plt.subplots(1, B, figsize=(B * 2.5, 2.5))
                if B == 1:
                    axes = [axes]
                for i in range(B):
                    axes[i].imshow(arr[i].clip(0, 1))
                    axes[i].axis("off")
                plt.tight_layout()
                plt.savefig(filename)
                plt.close(fig)
[content] /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/utils/dataloader.py (python)
def _create_processed_dataset_from_file(file_path, image_h, image_w, image_c, seq_len, num_parallel_calls):
    """Creates a fully processed dataset from a single TFRecord file."""
    dataset = tf.data.TFRecordDataset([file_path])
    
    parse_fn = functools.partial(
        _parse_tfrecord_fn, image_h=image_h, image_w=image_w, image_c=image_c
    )
    dataset = dataset.map(parse_fn, num_parallel_calls=num_parallel_calls)

    tf_process_fn = functools.partial(
        _tf_process_episode,
        seq_len=seq_len,
        image_h=image_h,
        image_w=image_w,
        image_c=image_c,
        seed=seed,
    )
    dataset = dataset.map(tf_process_fn, num_parallel_calls=num_parallel_calls)
    
    return dataset


def get_dataloader(
    tfrecord_paths: list[str],
    seq_len: int,
    global_batch_size: int,
    image_h: int,
    image_w: int,
    image_c: int,
    shuffle_buffer_size: int = 1000,
    num_parallel_calls: int = tf.data.AUTOTUNE,
    seed: int = 42,
    cycle_length: int = 4,
    block_length: int = 1,
):
    """
    Creates a tf.data.Dataset pipeline from TFRecord files.
    """
    if not tfrecord_paths:
        raise ValueError("tfrecord_paths list cannot be empty.")

    process_id = jax.process_index()
    num_processes = jax.process_count()

    assert (
        global_batch_size % num_processes == 0
    ), f"Global batch size {global_batch_size} \
        must be divisible by the number of JAX processes {num_processes} for proper sharding."
    per_process_batch_size = global_batch_size // num_processes

    def dataset_fn(file_path):
        return _create_processed_dataset_from_file(
            file_path, image_h, image_w, image_c, seq_len, num_parallel_calls
[content] /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/utils/dataloader.py (python)
def _create_processed_dataset_from_file(file_path, image_h, image_w, image_c, seq_len, num_parallel_calls, seed):
    """Creates a fully processed dataset from a single TFRecord file."""
    dataset = tf.data.TFRecordDataset([file_path])
    
    parse_fn = functools.partial(
        _parse_tfrecord_fn, image_h=image_h, image_w=image_w, image_c=image_c
    )
    dataset = dataset.map(parse_fn, num_parallel_calls=num_parallel_calls)

    tf_process_fn = functools.partial(
        _tf_process_episode,
        seq_len=seq_len,
        image_h=image_h,
        image_w=image_w,
        image_c=image_c,
        seed=seed,
    )
    dataset = dataset.map(tf_process_fn, num_parallel_calls=num_parallel_calls)
    
    return dataset


def get_dataloader(
    tfrecord_paths: list[str],
    seq_len: int,
    global_batch_size: int,
    image_h: int,
    image_w: int,
    image_c: int,
    shuffle_buffer_size: int = 10,
    num_parallel_calls: int = tf.data.AUTOTUNE,
    seed: int = 42,
    cycle_length: int = 4,
    block_length: int = 1,
):
    """
    Creates a tf.data.Dataset pipeline from TFRecord files.
    """
    if not tfrecord_paths:
        raise ValueError("tfrecord_paths list cannot be empty.")

    process_id = jax.process_index()
    num_processes = jax.process_count()

    assert (
        global_batch_size % num_processes == 0
    ), f"Global batch size {global_batch_size} \
        must be divisible by the number of JAX processes {num_processes} for proper sharding."
    per_process_batch_size = global_batch_size // num_processes

    def dataset_fn(file_path):
        return _create_processed_dataset_from_file(
            file_path, image_h, image_w, image_c, seq_len, num_parallel_calls, seed
[content] /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/utils/dataloader.py (python)
    # Filter out episodes that are too short
    def filter_short_episodes(episode_tensor):
        return tf.shape(episode_tensor)[0] >= seq_len
    
    dataset = dataset.filter(filter_short_episodes)
[tab] scripts_horeka/batchsize_scaling/adjusted_lr/tester.sh (shellscript)
#!/usr/bin/env bash

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name="debug"
slurm_job_id="0000"

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log \
    --name=tokenizer-batch-size-scaling-1-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 1-node time-step debug \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[content] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_default.sbatch (plaintext)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_1_node
#SBATCH --mem=100G

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --min_lr=3e-4 \
    --max_lr=3e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-1-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 1-node time-step \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[content] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_default.sbatch (plaintext)
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log \
    --name=tokenizer-batch-size-scaling-1-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 1-node debug \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --codebook_dropout 0.0
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_default copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_default

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log \
    --name=tokenizer-batch-size-scaling-1-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 1-node debug \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_37M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_default

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log \
    --name=tokenizer-batch-size-scaling-1-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 1-node debug \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_80M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_default

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log \
    --name=tokenizer-batch-size-scaling-1-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 1-node debug \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_80M copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=3
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_default

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-L1-$slurm_job_id \
    --tags tokenizer model-size-scaling L1 80M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=768 \
    --num_blocks=12 \
    --num_heads=12 \
    --latent_dim=64 \
    --num_latents=2048 \
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_140M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=3
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_default

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-L1-$slurm_job_id \
    --tags tokenizer model-size-scaling L1 80M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=768 \
    --num_blocks=12 \
    --num_heads=12 \
    --latent_dim=64 \
    --num_latents=2048 \
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_140M copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_140M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-L1-$slurm_job_id \
    --tags tokenizer model-size-scaling L1 80M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=768 \
    --num_blocks=12 \
    --num_heads=12 \
    --latent_dim=64 \
    --num_latents=2048 \
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_200.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_140M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-L1-$slurm_job_id \
    --tags tokenizer model-size-scaling L1 80M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=768 \
    --num_blocks=12 \
    --num_heads=12 \
    --latent_dim=64 \
    --num_latents=2048 \
[tab] scripts_horeka/modelsize_scaling/tokenizer/train_tokenizer_200M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_model_size_scaling_140M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=tokenizer-modelsize-L1-$slurm_job_id \
    --tags tokenizer model-size-scaling L1 80M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=768 \
    --num_blocks=12 \
    --num_heads=12 \
    --latent_dim=64 \
    --num_latents=2048 \
[tab] scripts_horeka/overfit_batch_tiny/train_dynamics_overfit_batch_big_lr.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --mail-user=mihir.mahajan2002@gmail.com
#SBATCH --job-name=train_dynamics_minecraft_overfit_sample_tiny
#SBATCH --mem=50G
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

echo Running dynamics model overfit run. Slurm id: $slurm_job_id

# Use checkpoints from tokenizer/lam overfit sample runs
tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3309662/tokenizer_1751381446_14500/
lam_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3309663/lam_1751381446_200000

srun python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log \
    --log_checkpoint_interval=500 \
    --name=masklim-0-dynamics-tiny-overfit-big-lr-gaussian-noise-$slurm_job_id \
    --tags dynamics overfit tiny masklimit-0 gaussian-noise \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --lam_checkpoint=$lam_ckpt_dir \
    --data_dir $tf_records_dir \
    --tokenizer_dim=384 \
    --latent_patch_dim=32 \
    --num_patch_latents=1024 \
    --patch_size=4 \
    --tokenizer_num_blocks=8 \
    --tokenizer_num_heads=8 \
    --lam_dim=384 \
    --latent_action_dim=32 \
    --num_latent_actions=6 \
    --lam_patch_size=16 \
    --lam_num_blocks=8 \
    --lam_num_heads=8 \
    --dyna_dim=128 \
    --dyna_num_blocks=2 \
    --dyna_num_heads=4 \
    --mask_limit=0.0 \
    --num_latent_actions=1
[tab] scripts_horeka/overfit_sample_tiny/train_dynamics_overfit_sample_big_lr-2.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --mail-user=mihir.mahajan2002@gmail.com
#SBATCH --job-name=train_dynamics_minecraft_overfit_sample_tiny
#SBATCH --mem=50G
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

echo Running dynamics model overfit run. Slurm id: $slurm_job_id

# Use checkpoints from tokenizer/lam overfit sample runs
tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3299272/tokenizer_1751037678_153500/
lam_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3299259/lam_1751036759_200000/

srun python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log \
    --log_checkpoint_interval=500 \
    --name=masklim-1-dynamics-tiny-overfit-big-lr-$slurm_job_id \
    --tags dynamics overfit tiny masklimit-1 \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --lam_checkpoint=$lam_ckpt_dir \
    --data_dir $tf_records_dir \
    --tokenizer_dim=384 \
    --latent_patch_dim=32 \
    --num_patch_latents=1024 \
    --patch_size=4 \
    --tokenizer_num_blocks=8 \
    --tokenizer_num_heads=8 \
    --lam_dim=384 \
    --latent_action_dim=32 \
    --num_latent_actions=6 \
    --lam_patch_size=16 \
    --lam_num_blocks=8 \
    --lam_num_heads=8 \
    --dyna_dim=128 \
    --dyna_num_blocks=2 \
    --dyna_num_heads=4 \
    --mask_limit=1.0
[tab] scripts_horeka/overfit_sample_tiny/train_lam_overfit_sample.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --mail-user=mihir.mahajan2002@gmail.com
#SBATCH --job-name=train_lam_minecraft_overfit_sample
#SBATCH --mem=50G
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=3e-4 \
    --max_lr=3e-4 \
    --warmup_steps=125 \
    --log_image_interval=3 \
    --log \
    --log_checkpoint_interval=500 \
    --name=lam-tiny-overfit-$slurm_job_id \
    --tags lam overfit tiny \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=384 \
    --latent_dim=32 \
    --num_latents=6 \
    --patch_size=16 \
    --num_blocks=8 \
    --num_heads=8 \
    --codebook_dropout=0.0
[tab] scripts_horeka/batchsize_scaling/adjusted_lr/train_tokenizer_4_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_4_node
#SBATCH --mem=100G

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=192 \
    --min_lr=6.00e-4 \
    --max_lr=6.00e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-4-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 4-node time-step \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/batchsize_scaling/adjusted_lr/train_tokenizer_2_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_2_node
#SBATCH --mem=100G

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=4.24e-4 \
    --max_lr=4.24e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-2-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 2-node time-step \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir