p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] utils/dataloader.py (python)
def _create_processed_dataset_from_file(file_path, image_h, image_w, image_c, seq_len, num_parallel_calls):
    """Creates a fully processed dataset from a single TFRecord file."""
    dataset = tf.data.TFRecordDataset([file_path])
[tab] slurm/jobs/mihir/horeka/batchsize_scaling/adjusted_lr/train_tokenizer_1_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_1_node
#SBATCH --mem=100G

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --min_lr=3e-4 \
    --max_lr=3e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-1-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 1-node time-step \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] slurm/jobs/mihir/horeka/batchsize_scaling/adjusted_lr/train_tokenizer_2_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_2_node
#SBATCH --mem=100G

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=4.24e-4 \
    --max_lr=4.24e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-2-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 2-node time-step \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] slurm/jobs/mihir/horeka/batchsize_scaling/adjusted_lr/train_tokenizer_8_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_8_node
#SBATCH --mem=100G

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=384 \
    --min_lr=8.49e-4 \
    --max_lr=8.49e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-8-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 8-node time-step \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] slurm/jobs/mihir/horeka/batchsize_scaling/adjusted_lr/train_tokenizer_32_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=32
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_32_node
#SBATCH --mem=400G

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1536 \
    --min_lr=3e-4 \
    --max_lr=3e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-32-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 32-node \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] slurm/jobs/mihir/horeka/batchsize_scaling/adjusted_lr/train_tokenizer_16_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=16
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_16_node
#SBATCH --mem=200G

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=768 \
    --min_lr=1.20e-3 \
    --max_lr=1.20e-3 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-16-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 16-node \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] slurm/jobs/mihir/horeka/batchsize_scaling/adjusted_lr/train_tokenizer_4_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_4_node
#SBATCH --mem=100G

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=192 \
    --min_lr=6.00e-4 \
    --max_lr=6.00e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-4-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 4-node time-step \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/overfit_sample_tiny/sample.sh (shellscript)
#!/usr/bin/env bash

# Unload modules that may interfere
module unload mpi/openmpi/5.0
module unload devel/cuda/12.4

# Activate virtual environment
source .venv/bin/activate

# Set workspace and checkpoint directory (update slurm_job_id as needed)
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared'
# Replace the following with the actual job id/checkpoint you want to sample from
slurm_job_id=3301031

# job_name=train_dynamics_minecraft_overfit_sample_tiny
CHECKPOINT_DIR=$ws_dir/checkpoints/${slurm_job_id}

# Example: If you want to use a specific checkpoint, set it here
# CHECKPOINT_PATH=$ws_dir/checkpoints/3299272/dynamics-tiny-overfit-big-lr-3299272_50000/
# Or use the latest in the directory
# CHECKPOINT_PATH=$(ls -d $CHECKPOINT_DIR/*/ | sort | tail -n 1)
CHECKPOINT_PATH=$CHECKPOINT_DIR/genie_1751067778_200000/

echo "Sampling from checkpoint: $CHECKPOINT_PATH"

python sample.py \
    --checkpoint "$CHECKPOINT_PATH" \
    --tokenizer_dim=384 \
    --latent_patch_dim=32 \
    --num_patch_latents=1024 \
    --patch_size=4 \
    --tokenizer_num_blocks=8 \
    --tokenizer_num_heads=8 \
    --lam_dim=384 \
    --latent_action_dim=32 \
    --num_latent_actions=6 \
    --lam_patch_size=16 \
    --lam_num_blocks=8 \
    --lam_num_heads=8 \
    --dyna_dim=128 \
    --dyna_num_blocks=2 \
    --dyna_num_heads=4
[content] sample.py (python)
#from utils.dataloader import get_dataloader


@dataclass
class Args:
    # Experiment
    seed: int = 0
    seq_len: int = 16
    image_channels: int = 3
    image_resolution: int = 64
    data_dir: str = "data/coinrun_episodes"
    checkpoint: str = ""
    # Sampling
    batch_size: int = 1
    maskgit_steps: int = 25
    temperature: float = 1.0
    sample_argmax: bool = True
    start_frame: int = 0
    # Tokenizer checkpoint
    tokenizer_dim: int = 512
    latent_patch_dim: int = 32
    num_patch_latents: int = 1024
    patch_size: int = 4
    tokenizer_num_blocks: int = 8
    tokenizer_num_heads: int = 8
    # LAM checkpoint
    lam_dim: int = 512
    latent_action_dim: int = 32
    num_latent_actions: int = 6
    lam_patch_size: int = 16
    lam_num_blocks: int = 8
    lam_num_heads: int = 8
    # Dynamics checkpoint
    dyna_dim: int = 512
    dyna_num_blocks: int = 12
    dyna_num_heads: int = 8


args = tyro.cli(Args)
rng = jax.random.PRNGKey(args.seed)

# --- Load Genie checkpoint ---
genie = Genie(
    # Tokenizer
    in_dim=args.image_channels,
    tokenizer_dim=args.tokenizer_dim,
    latent_patch_dim=args.latent_patch_dim,
    num_patch_latents=args.num_patch_latents,
    patch_size=args.patch_size,
    tokenizer_num_blocks=args.tokenizer_num_blocks,
    tokenizer_num_heads=args.tokenizer_num_heads,
    # LAM
    lam_dim=args.lam_dim,
    latent_action_dim=args.latent_action_dim,
    num_latent_actions=args.num_latent_actions,
    lam_patch_size=args.lam_patch_size,
    lam_num_blocks=args.lam_num_blocks,
    lam_num_heads=args.lam_num_heads,
    # Dynamics
    dyna_dim=args.dyna_dim,
    dyna_num_blocks=args.dyna_num_blocks,
    dyna_num_heads=args.dyna_num_heads,
)
rng, _rng = jax.random.split(rng)
image_shape = (args.image_resolution, args.image_resolution, args.image_channels)
dummy_inputs = dict(
    videos=jnp.zeros((args.batch_size, args.seq_len, *image_shape), dtype=jnp.float32),
    mask_rng=_rng,
)
rng, _rng = jax.random.split(rng)
params = genie.init(_rng, dummy_inputs)
ckpt = PyTreeCheckpointer().restore(args.checkpoint)["model"]["params"]["params"]
params["params"].update(ckpt)


# --- Define autoregressive sampling loop ---
def _autoreg_sample(rng, video_batch, action_batch):
    vid = video_batch[:, : args.start_frame + 1]
    for frame_idx in range(args.start_frame + 1, args.seq_len):
        # --- Sample next frame ---
        print("Frame", frame_idx)
        rng, _rng = jax.random.split(rng)
        batch = dict(videos=vid, latent_actions=action_batch[:, :frame_idx], rng=_rng)
        new_frame = genie.apply(
            params,
            batch,
            args.maskgit_steps,
            args.temperature,
            args.sample_argmax,
            method=Genie.sample,
        )
        vid = jnp.concatenate([vid, new_frame], axis=1)
    return vid


# --- Get video + latent actions ---
# dataloader = get_dataloader(args.data_dir, args.seq_len, args.batch_size)
# video_batch = next(iter(dataloader))
video_batch = np.load("overfit_dir/single_sample_corner.npy")
# Get latent actions from first video only
first_video = video_batch[:1, :args.seq_len]
[tab] scripts_horeka/overfit_sample_tiny/sample.sh (shellscript)
#!/usr/bin/env bash

# Unload modules that may interfere
module unload mpi/openmpi/5.0
module unload devel/cuda/12.4

# Activate virtual environment
source .venv/bin/activate

# Set workspace and checkpoint directory (update slurm_job_id as needed)
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared'
# Replace the following with the actual job id/checkpoint you want to sample from
slurm_job_id=3301029

# job_name=train_dynamics_minecraft_overfit_sample_tiny
CHECKPOINT_DIR=$ws_dir/checkpoints/${slurm_job_id}

# Example: If you want to use a specific checkpoint, set it here
# CHECKPOINT_PATH=$ws_dir/checkpoints/3299272/dynamics-tiny-overfit-big-lr-3299272_50000/
# Or use the latest in the directory
# CHECKPOINT_PATH=$(ls -d $CHECKPOINT_DIR/*/ | sort | tail -n 1)
CHECKPOINT_PATH=$CHECKPOINT_DIR/genie_1751067601_200000/

echo "Sampling from checkpoint: $CHECKPOINT_PATH"

python sample.py \
    --checkpoint "$CHECKPOINT_PATH" \
    --tokenizer_dim=384 \
    --latent_patch_dim=32 \
    --num_patch_latents=1024 \
    --patch_size=4 \
    --tokenizer_num_blocks=8 \
    --tokenizer_num_heads=8 \
    --lam_dim=384 \
    --latent_action_dim=32 \
    --num_latent_actions=6 \
    --lam_patch_size=16 \
    --lam_num_blocks=8 \
    --lam_num_heads=8 \
    --dyna_dim=128 \
    --dyna_num_blocks=2 \
    --dyna_num_heads=4 \
    --maskgit_steps=1
[tab] scripts_horeka/train_dynamics.sh (shellscript)
#!/usr/bin/env bash

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate


tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'


job_name="debug"
slurm_job_id="0000"

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

tokenizer_ckpt_dir=$ws_dir/checkpoints/3290391/tokenizer_1750845012_40000
lam_ckpt_dir=$ws_dir/checkpoints/3290392/lam_1750845133_130000
python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=5e-7 \
    --max_lr=5e-6 \
    --warmup_steps=125 \
    --log_image_interval=100 \
    --log \
    --log_checkpoint_interval=500 \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --lam_checkpoint=$lam_ckpt_dir \
    --data_dir $tf_records_dir
[tab] scripts_horeka/overfit_sample_tiny/train_dynamics_overfit_sample_big_lr.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --mail-user=mihir.mahajan2002@gmail.com
#SBATCH --job-name=train_dynamics_minecraft_overfit_sample_tiny
#SBATCH --mem=50G
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

echo Running dynamics model overfit run. Slurm id: $slurm_job_id

# Use checkpoints from tokenizer/lam overfit sample runs
tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3299272/tokenizer_1751037678_153500/
lam_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3299259/lam_1751036759_200000/

srun python train_dynamics.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log \
    --log_checkpoint_interval=500 \
    --name=dynamics-tiny-overfit-big-lr-$slurm_job_id \
    --tags dynamics overfit tiny \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --lam_checkpoint=$lam_ckpt_dir \
    --data_dir $tf_records_dir \
    --tokenizer_dim=384 \
    --latent_patch_dim=32 \
    --num_patch_latents=1024 \
    --patch_size=4 \
    --tokenizer_num_blocks=8 \
    --tokenizer_num_heads=8 \
    --lam_dim=384 \
    --latent_action_dim=32 \
    --num_latent_actions=6 \
    --lam_patch_size=16 \
    --lam_num_blocks=8 \
    --lam_num_heads=8 \
    --dyna_dim=128 \
    --dyna_num_blocks=2 \
    --dyna_num_heads=4
[content] scripts_horeka/train_dynamics.sh (shellscript)
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log \
    --log_checkpoint_interval=500 \
    --name=dynamics-tiny-overfit-big-lr-$slurm_job_id \
    --tags dynamics overfit tiny \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --lam_checkpoint=$lam_ckpt_dir \
    --data_dir $tf_records_dir \
    --tokenizer_dim=384 \
    --latent_patch_dim=32 \
    --num_patch_latents=1024 \
    --patch_size=4 \
    --tokenizer_num_blocks=8 \
    --tokenizer_num_heads=8 \
    --lam_dim=384 \
    --latent_action_dim=32 \
    --num_latent_actions=6 \
    --lam_patch_size=16 \
    --lam_num_blocks=8 \
    --lam_num_heads=8 \
    --dyna_dim=128 \
    --dyna_num_blocks=2 \
    --dyna_num_heads=4
[content] scripts_horeka/train_dynamics.sh (shellscript)
# Use checkpoints from tokenizer/lam overfit sample runs
tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3299272/tokenizer_1751037678_153500/
lam_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/3299259/lam_1751036759_200000/
[tab] scripts_horeka/train_lam.sh (shellscript)
#!/usr/bin/env bash

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate


tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name="debug"
slurm_job_id="0000"

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=5e-7 \
    --max_lr=5e-6 \
    --warmup_steps=125 \
    --log_image_interval=3 \
    --log \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[tab] scripts_horeka/overfit_sample_tiny/train_lam_overfit_sample.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=15:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --mail-user=mihir.mahajan2002@gmail.com
#SBATCH --job-name=train_lam_minecraft_overfit_sample
#SBATCH --mem=50G
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=3e-4 \
    --max_lr=3e-4 \
    --warmup_steps=125 \
    --log_image_interval=3 \
    --log \
    --log_checkpoint_interval=500 \
    --name=lam-tiny-overfit-$slurm_job_id \
    --tags lam overfit tiny \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=384 \
    --latent_dim=32 \
    --num_latents=6 \
    --patch_size=16 \
    --num_blocks=8 \
    --num_heads=8 \
    --codebook_dropout=0.0
[content] scripts_horeka/train_lam.sh (shellscript)
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=3e-4 \
    --max_lr=3e-4 \
    --warmup_steps=125 \
    --log_image_interval=3 \
    --log \
    --log_checkpoint_interval=500 \
    --name=lam-tiny-overfit-$slurm_job_id \
    --tags lam overfit tiny \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir \
    --model_dim=384 \
    --latent_dim=32 \
    --num_latents=6 \
    --patch_size=16 \
    --num_blocks=8 \
    --num_heads=8 \
    --codebook_dropout=0.0
[content] train_dynamics.py (python)

                        log_images = dict(
                            image=wandb.Image(np.asarray(gt_seq[args.seq_len - 1])),
                            recon=wandb.Image(np.asarray(recon_seq[args.seq_len - 1])),
                            true_vs_recon=wandb.Image(
                                np.asarray(comparison_seq.astype(np.uint8))
                            ),
                        )
                        wandb.log(log_images)