p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] train_lam.py (python)
@nnx.jit
def val_step(lam: LatentActionModel, inputs: dict) -> tuple[jax.Array, jax.Array, dict]:
    lam.eval()
    (loss, (recon, idx_counts, metrics)) = lam_loss_fn(lam, inputs)
    return loss, recon, metrics
[content] train_lam.py (python)
def calculate_validation_metrics(val_dataloader):
    step = 0
    loss_per_step = []
    metrics_per_step = []
    print(f"Calculating validation metrics...")
    for videos in val_dataloader:
        inputs = dict(videos=videos)
        loss, recon, metrics = val_step(lam, inputs)
        loss_per_step.append(loss)
        metrics_per_step.append(metrics)
        step += 1
        if step > args.val_steps:
            break

    if step < args.val_steps:
        print(f"Warning: Your validation dataset is too small to make val_steps many steps. Made {step} steps, expected {args.val_steps}")

    val_loss = np.mean(loss_per_step)
    val_metrics = {
        f"val_{key}": np.mean([m[key] for m in metrics_per_step])
        for key in metrics_per_step[0].keys()
    }
    return val_loss, recon, val_metrics
[content] train_lam.py (python)
            # --- Validation loss ---
            if args.val_data_dir and step % args.val_interval == 0:
                print(f"Calculating validation metrics...")
                val_loss, val_recon, val_metrics = calculate_validation_metrics(dataloader_val)
                print(f"Step {step}, validation loss: {val_loss}")

            # --- Logging ---
            if args.log:
                if step % args.log_interval == 0 and jax.process_index() == 0:
                    log_dict = {
                        "loss": loss,
                        "step": step,
                        **metrics
                        }
                    if args.val_data_dir and step % args.val_interval == 0:
                        log_dict.update({
                            "val_loss": val_loss,
                            **val_metrics
                        })
                    wandb.log(log_dict)
[content] train_lam.py (python)
    dataloader_train = (
        jax.make_array_from_process_local_data(videos_sharding, elem)
        for elem in train_iterator
    )
    dataloader_val = (
        jax.make_array_from_process_local_data(videos_sharding, elem)
        for elem in val_iterator
    )
[content] train_lam.py (python)
    # --- Create DataLoaderIterator from dataloader ---
    image_shape = (args.image_height, args.image_width, args.image_channels)
    train_iterator = create_dataloader_iterator(args.data_dir, image_shape, args.seq_len, args.batch_size, args.seed)
    val_iterator = create_dataloader_iterator(args.val_data_dir, image_shape, args.seq_len, args.batch_size, args.seed)
[content] train_lam.py (python)
    handler_registry.add(
        "train_dataloader_state",
        grain.checkpoint.CheckpointSave,
        cast(ocp.handlers.CheckpointHandler, grain.checkpoint.CheckpointHandler),
    )
    handler_registry.add(
        "train_dataloader_state",
        grain.checkpoint.CheckpointRestore,
        cast(ocp.handlers.CheckpointHandler, grain.checkpoint.CheckpointHandler),
    )
    handler_registry.add(
        "val_dataloader_state",
        grain.checkpoint.CheckpointSave,
        cast(ocp.handlers.CheckpointHandler, grain.checkpoint.CheckpointHandler),
    )
    handler_registry.add(
        "val_dataloader_state",
        grain.checkpoint.CheckpointRestore,
        cast(ocp.handlers.CheckpointHandler, grain.checkpoint.CheckpointHandler),
    )
[content] train_lam.py (python)
                    args=ocp.args.Composite(
                        model_state=ocp.args.PyTreeSave(optimizer_state),  # type: ignore
                        train_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            train_iterator  # type: ignore
                        ),
                        val_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            val_iterator  # type: ignore
                        ),
                    ),
[content] train_tokenizer.py (python)
[0].astype(jnp.float32) / 255.0
                    recon_seq = recon[0].clip(0, 1)
                    comparison_seq = jnp.concatenate((gt_seq, recon_seq), axis=1)
                    comparison_seq = einops.rearrange(
                        comparison_seq * 255, "t h w c -> h (t w) c"
                    )
[content] train_tokenizer.py (python)

                            image=wandb.Image(np.asarray(gt_seq[0])),
                            recon=wandb.Image(np.asarray(recon_seq[0])),
                            true_vs_recon=wandb.Image(
                                np.asarray(comparison_seq.astype(np.uint8))
                            ),
[content] train_lam.py (python)
                if step % args.log_image_interval == 0:
                    gt_seq = inputs["videos"][0].astype(jnp.float32) / 255.0
                    recon_seq = recon[0].clip(0, 1)
                    comparison_seq = jnp.concatenate((gt_seq, recon_seq), axis=1)
                    comparison_seq = einops.rearrange(
                        comparison_seq * 255, "t h w c -> h (t w) c"
                    )
                    gt_seq_val = val_gt_batch["videos"][0].astype(jnp.float32) / 255.0
                    recon_seq_val = val_recon[0].clip(0, 1)
                    val_comparison_seq = jnp.concatenate((gt_seq, recon_seq), axis=1)
                    val_comparison_seq = einops.rearrange(
                        val_comparison_seq * 255, "t h w c -> h (t w) c"
                    )
                    # NOTE: Process-dependent control flow deliberately happens
                    # after indexing operation since it must not contain code
                    # sections that lead to cross-accelerator communication.
                    if jax.process_index() == 0:
                        log_images = dict(
                            image=wandb.Image(np.asarray(gt_seq[0])),
                            recon=wandb.Image(np.asarray(recon_seq[0])),
                            true_vs_recon=wandb.Image(
                                np.asarray(comparison_seq.astype(np.uint8))
                            ),
                            val_image=wandb.Image(np.asarray(gt_seq_val[0])),
                            val_recon=wandb.Image(np.asarray(recon_seq_val[0])),
                            val_true_vs_recon=wandb.Image(
                                np.asarray(val_comparison_seq.astype(np.uint8))
                            ),
                        )
                        wandb.log(log_images)
[tab] slurm/jobs/mihir/horeka/coinrun/train_tokenizer_single_gpu.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_1e-4
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --save_ckpt \
    --image_height=64 \
    --image_width=64 \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=coinrun-tokenizer-dev-$slurm_job_id \
    --tags tokenizer coinrun dev \
    --entity instant-uv \
    --project jafar \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --num_steps 200 \
    --val_data_dir $array_records_dir_val \
    --val_interval 10 \
    --val_steps 50 \
    --data_dir $array_records_dir_train &

child_pid=$!

wait $child_pid
[tab] slurm/jobs/mihir/horeka/coinrun/train_tokenizer_single_gpu copy.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_1e-4
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --save_ckpt \
    --image_height=64 \
    --image_width=64 \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=coinrun-tokenizer-dev-$slurm_job_id \
    --tags tokenizer coinrun dev \
    --entity instant-uv \
    --project jafar \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --num_steps 200 \
    --val_data_dir $array_records_dir_val \
    --val_interval 10 \
    --val_steps 50 \
    --data_dir $array_records_dir_train &

child_pid=$!

wait $child_pid
[tab] slurm/jobs/mihir/horeka/coinrun/train_lam_single_gpu.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_1e-4
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --save_ckpt \
    --image_height=64 \
    --image_width=64 \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=coinrun-tokenizer-dev-$slurm_job_id \
    --tags tokenizer coinrun dev \
    --entity instant-uv \
    --project jafar \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --num_steps 200 \
    --val_data_dir $array_records_dir_val \
    --val_interval 10 \
    --val_steps 50 \
    --data_dir $array_records_dir_train &

child_pid=$!

wait $child_pid
[content] /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/train_lam.py (python)

                            val_image=wandb.Image(np.asarray(gt_seq_val[0])),
                            val_recon=wandb.Image(np.asarray(recon_seq_val[0])),
                            val_true_vs_recon=wandb.Image(
                                np.asarray(val_comparison_seq.astype(np.uint8))
                            ),
[content] /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/train_lam.py (python)
            args=ocp.args.Composite(
                model_state=ocp.args.PyTreeRestore(abstract_optimizer_state),  # type: ignore
                dataloader_state=grain.checkpoint.CheckpointRestore(train_iterator),  # type: ignore
                val_dataloader_state=grain.checkpoint.CheckpointRestore(val_iterator),  # type: ignore
            ),
[content] /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/train_lam.py (python)

            restore_args = ocp.args.Composite(
                    model_state=ocp.args.PyTreeRestore(abstract_optimizer_state),  # type: ignore
                    dataloader_state=grain.checkpoint.CheckpointRestore(train_iterator),  # type: ignore
                    val_dataloader_state=grain.checkpoint.CheckpointRestore(val_iterator),  # type: ignore
                )
[content] /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/train_lam.py (python)

                    args=ocp.args.Composite(
                        model_state=ocp.args.PyTreeSave(optimizer_state),  # type: ignore
                        train_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            train_iterator  # type: ignore
                        ),
                        val_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            val_iterator  # type: ignore
                        ),
[content] /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/train_lam.py (python)

                    ckpt_manager_args = ocp.args.Composite(
                        model_state=ocp.args.PyTreeSave(optimizer_state),  # type: ignore
                        train_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            train_iterator  # type: ignore
                        ),
                        val_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            val_iterator  # type: ignore
                        )
                    )
[tab] slurm/jobs/mihir/horeka/coinrun/train_lam_single_gpu.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=00:20:00
#SBATCH --partition=dev_accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/lam/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/lam/%x_%j.log
#SBATCH --job-name=train_lam_1e-4
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/lam/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --save_ckpt \
    --image_height=64 \
    --image_width=64 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=coinrun-lam-dev-$slurm_job_id \
    --tags lam coinrun dev \
    --entity instant-uv \
    --project jafar \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --num_steps 200 \
    --val_interval 10 \
    --val_steps 50 \
    --data_dir $array_records_dir_train
[tab] slurm/jobs/mihir/horeka/coinrun/train_lam_single_gpu.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=00:20:00
#SBATCH --partition=dev_accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/lam/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/lam/%x_%j.log
#SBATCH --job-name=train_lam_1e-4
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/lam/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --save_ckpt \
    --image_height=64 \
    --image_width=64 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=coinrun-lam-dev-$slurm_job_id \
    --tags lam coinrun dev \
    --entity instant-uv \
    --project jafar \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --num_steps 200 \
    --data_dir $array_records_dir_train
[tab] slurm/jobs/mihir/horeka/coinrun/train_tokenizer_single_gpu.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=00:20:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_1e-4
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --save_ckpt \
    --image_height=64 \
    --image_width=64 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=coinrun-tokenizer-dev-$slurm_job_id \
    --tags tokenizer coinrun dev \
    --entity instant-uv \
    --project jafar \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --num_steps 200 \
    --val_data_dir $array_records_dir_val \
    --val_interval 10 \
    --val_steps 50 \
    --data_dir $array_records_dir_train
[content] train_tokenizer.py (python)
        if args.val_data_dir:
            restore_args = ocp.args.Composite(
                    model_state=ocp.args.PyTreeRestore(abstract_optimizer_state),  # type: ignore
                    dataloader_state=grain.checkpoint.CheckpointRestore(train_iterator),  # type: ignore
                    val_dataloader_state=grain.checkpoint.CheckpointRestore(val_iterator),  # type: ignore
                )
        else: 
            restore_args = ocp.args.Composite(
                    model_state=ocp.args.PyTreeRestore(abstract_optimizer_state),  # type: ignore
                    dataloader_state=grain.checkpoint.CheckpointRestore(train_iterator),  # type: ignore
                )
        restored = checkpoint_manager.restore(
            checkpoint_manager.latest_step(),
            args=restore_args
        )
[content] train_tokenizer.py (python)
                    if jax.process_index() == 0:
                        log_images = dict(
                            image=wandb.Image(np.asarray(gt_seq[0])),
                            recon=wandb.Image(np.asarray(recon_seq[0])),
                            true_vs_recon=wandb.Image(
                                np.asarray(comparison_seq.astype(np.uint8))
                            ),
                        )
                        if args.val_data_dir and step % args.val_interval == 0:
                            log_images.update(
                                dict(
                                    val_image=wandb.Image(np.asarray(gt_seq_val[0])),
                                    val_recon=wandb.Image(np.asarray(recon_seq_val[0])),
                                    val_true_vs_recon=wandb.Image(
                                        np.asarray(val_comparison_seq.astype(np.uint8))
                                    )
                                )
                            )
                        wandb.log(log_images)
[content] train_tokenizer.py (python)
                optimizer_state = nnx.state(optimizer)
                if args.val_data_dir:
                    ckpt_manager_args = ocp.args.Composite(
                        model_state=ocp.args.PyTreeSave(optimizer_state),  # type: ignore
                        train_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            train_iterator  # type: ignore
                        ),
                        val_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            val_iterator  # type: ignore
                        )
                    )
                else: 
                    ckpt_manager_args = ocp.args.Composite(
                        model_state=ocp.args.PyTreeSave(optimizer_state),  # type: ignore
                        train_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            train_iterator  # type: ignore
                        )
                    )
                checkpoint_manager.save(
                    step,
                    args=ckpt_manager_args
                    ),
[content] train_dynamics.py (python)
    handler_registry.add(
        "train_dataloader_state",
        grain.checkpoint.CheckpointSave,
        cast(ocp.handlers.CheckpointHandler, grain.checkpoint.CheckpointHandler),
    )
    handler_registry.add(
        "train_dataloader_state",
        grain.checkpoint.CheckpointRestore,
        cast(ocp.handlers.CheckpointHandler, grain.checkpoint.CheckpointHandler),
    )
    if args.val_data_dir:
        handler_registry.add(
            "val_dataloader_state",
            grain.checkpoint.CheckpointSave,
            cast(ocp.handlers.CheckpointHandler, grain.checkpoint.CheckpointHandler),
        )
        handler_registry.add(
            "val_dataloader_state",
            grain.checkpoint.CheckpointRestore,
            cast(ocp.handlers.CheckpointHandler, grain.checkpoint.CheckpointHandler),
        )
[content] train_dynamics.py (python)
    train_iterator = create_dataloader_iterator(args.data_dir, image_shape, args.seq_len, args.batch_size, args.seed)
    if args.val_data_dir:
        val_iterator = create_dataloader_iterator(args.val_data_dir, image_shape, args.seq_len, args.batch_size, args.seed)
[content] train_dynamics.py (python)
        if args.val_data_dir:
            restore_args = ocp.args.Composite(
                    model_state=ocp.args.PyTreeRestore(abstract_optimizer_state),  # type: ignore
                    dataloader_state=grain.checkpoint.CheckpointRestore(train_iterator),  # type: ignore
                    val_dataloader_state=grain.checkpoint.CheckpointRestore(val_iterator),  # type: ignore
                )
        else: 
            restore_args = ocp.args.Composite(
                    model_state=ocp.args.PyTreeRestore(abstract_optimizer_state),  # type: ignore
                    dataloader_state=grain.checkpoint.CheckpointRestore(train_iterator),  # type: ignore
                )
        restored = checkpoint_manager.restore(
            checkpoint_manager.latest_step(),
            args=restore_args
        )
[content] train_dynamics.py (python)
    dataloader_train = (
        jax.make_array_from_process_local_data(videos_sharding, elem)
        for elem in train_iterator
    )
    if args.val_data_dir:
        dataloader_val = (
            jax.make_array_from_process_local_data(videos_sharding, elem)
            for elem in val_iterator
        )
[content] train_dynamics.py (python)
            # --- Validation loss ---
            if args.val_data_dir and step % args.val_interval == 0:
                print(f"Calculating validation metrics...")
                val_loss, val_metrics, val_gt_batch, val_recon = calculate_validation_metrics(dataloader_val)
                print(f"Step {step}, validation loss: {val_loss}")

            # --- Logging ---
            if args.log:
                if step % args.log_interval == 0 and jax.process_index() == 0:
                    log_dict = {
                        "loss": loss,
                        "step": step,
                        **metrics
                        }
                    if args.val_data_dir and step % args.val_interval == 0:
                        log_dict.update({
                            "val_loss": val_loss,
                            **val_metrics
                        })
                    wandb.log(log_dict)
                if step % args.log_image_interval == 0:
                    gt_seq = inputs["videos"][0].astype(jnp.float32) / 255.0
                    recon_seq = recon[0].clip(0, 1)
                    comparison_seq = jnp.concatenate((gt_seq, recon_seq), axis=1)
                    comparison_seq = einops.rearrange(
                        comparison_seq * 255, "t h w c -> h (t w) c"
                    )
                    if args.val_data_dir and step % args.val_interval == 0:
                        gt_seq_val = val_gt_batch["videos"][0].astype(jnp.float32) / 255.0
                        recon_seq_val = val_recon[0].clip(0, 1)
                        val_comparison_seq = jnp.concatenate((gt_seq, recon_seq), axis=1)
                        val_comparison_seq = einops.rearrange(
                            val_comparison_seq * 255, "t h w c -> h (t w) c"
                        )
                    # NOTE: Process-dependent control flow deliberately happens
                    # after indexing operation since it must not contain code
                    # sections that lead to cross-accelerator communication.
                    if jax.process_index() == 0:
                        log_images = dict(
                            image=wandb.Image(np.asarray(gt_seq[0])),
                            recon=wandb.Image(np.asarray(recon_seq[0])),
                            true_vs_recon=wandb.Image(
                                np.asarray(comparison_seq.astype(np.uint8))
                            ),
                        )
                        if args.val_data_dir and step % args.val_interval == 0:
                            log_images.update(
                                dict(
                                    val_image=wandb.Image(np.asarray(gt_seq_val[0])),
                                    val_recon=wandb.Image(np.asarray(recon_seq_val[0])),
                                    val_true_vs_recon=wandb.Image(
                                        np.asarray(val_comparison_seq.astype(np.uint8))
                                    )
                                )
                            )
                        wandb.log(log_images)
[content] train_dynamics.py (python)
               optimizer_state = nnx.state(optimizer)
                if args.val_data_dir:
                    ckpt_manager_args = ocp.args.Composite(
                        model_state=ocp.args.PyTreeSave(optimizer_state),  # type: ignore
                        train_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            train_iterator  # type: ignore
                        ),
                        val_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            val_iterator  # type: ignore
                        )
                    )
                else: 
                    ckpt_manager_args = ocp.args.Composite(
                        model_state=ocp.args.PyTreeSave(optimizer_state),  # type: ignore
                        train_dataloader_state=grain.checkpoint.CheckpointSave(  # type: ignore
                            train_iterator  # type: ignore
                        )
                    )
                checkpoint_manager.save(
                    step,
                    args=ckpt_manager_args
                    ),
[content] train_dynamics.py (python)
def calculate_validation_metrics(val_dataloader):
    step = 0
    loss_per_step = []
    metrics_per_step = []
    for videos in val_dataloader:
        inputs = dict(videos=videos)
        loss, recon, metrics = val_step(tokenizer, inputs)
        loss_per_step.append(loss)
        metrics_per_step.append(metrics)
        step += 1
        if step > args.val_steps:
            break

    if step < args.val_steps:
        print(f"Warning: Your validation dataset is too small to make val_steps many steps. Made {step} steps, expected {args.val_steps}")

    val_loss = np.mean(loss_per_step)
    val_metrics = {
        f"val_{key}": np.mean([m[key] for m in metrics_per_step])
        for key in metrics_per_step[0].keys()
    }
    return val_loss, val_metrics, inputs, recon
[content] train_dynamics.py (python)
@nnx.jit
def val_step(tokenizer: TokenizerVQVAE, inputs: dict) -> tuple[jax.Array, jax.Array, dict]:
    tokenizer.eval()
    (loss, (recon, metrics)) = tokenizer_loss_fn(tokenizer, inputs)
    return loss, recon, metrics
[tab] slurm/jobs/mihir/horeka/coinrun/train_tokenizer_single_gpu copy.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=00:20:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_1e-4
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --save_ckpt \
    --image_height=64 \
    --image_width=64 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=coinrun-tokenizer-dev-$slurm_job_id \
    --tags tokenizer coinrun dev \
    --entity instant-uv \
    --project jafar \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --num_steps 200 \
    --val_data_dir $array_records_dir_val \
    --val_interval 10 \
    --val_steps 50 \
    --data_dir $array_records_dir_train
[tab] slurm/jobs/mihir/horeka/coinrun/train_dyn_single_gpu.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=00:20:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log
#SBATCH --job-name=train_tokenizer_1e-4
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --save_ckpt \
    --image_height=64 \
    --image_width=64 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=coinrun-tokenizer-dev-$slurm_job_id \
    --tags tokenizer coinrun dev \
    --entity instant-uv \
    --project jafar \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --num_steps 200 \
    --val_data_dir $array_records_dir_val \
    --val_interval 10 \
    --val_steps 50 \
    --data_dir $array_records_dir_train
[content] train_dynamics.py (python)
    for videos in val_dataloader:
        inputs = dict(videos=videos)
        loss, recon, metrics = val_step(genie, inputs)
        loss_per_step.append(loss)
        metrics_per_step.append(metrics)
[tab] slurm/jobs/mihir/horeka/lam/train_lam_minecraft_8node-darkness-filter-400M.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/lam/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/lam/%x_%j.log
#SBATCH --job-name=train_lam_minecraft_8node_darkness_filter_400M
#SBATCH --reservation=llmtum
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
# slurm_job_id=$SLURM_JOB_ID
slurm_job_id=3454954

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/lam/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --save_ckpt \
    --restore_ckpt \
    --wandb_id $slurm_job_id \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=160 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --darkness_threshold=50 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=lam-minecraft-8-node-darkness-filter-400M-$slurm_job_id \
    --tags lam minecraft 8-node darkness-filter 400M \
    --entity instant-uv \
    --project jafar \
    --num_latents=100 \
    --model_dim=1024 \
    --num_blocks=12 \
    --num_heads=16 \
    --latent_dim=64 \
    --ffn_dim=4096 \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
[content] slurm/dev/mihir/horeka/chunked_ablation/train_dynamics_chunked_dataset.sbatch (plaintext)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/maskgit/dynamics-cotraining/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/maskgit/dynamics-cotraining/%x_%j.log
#SBATCH --job-name=train_dynamics_maskgit_8_node
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi



# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/holiday/maskgit/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401

env | grep SLURM

srun python train_dynamics.py \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=160 \
    --init_lr=0 \
    --dyna_type=maskgit \
    --num_latent_actions=100 \
    --max_lr=8e-5 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=dynamics-maskgit-8-node-$slurm_job_id \
    --tags dynamics maskgit 8-node post-launch-main \
    --entity instant-uv \
    --project jafar \
    --dyna_dim=1024 \
    --dyna_num_blocks=16 \
    --dyna_num_heads=16 \
    --dyna_ffn_dim=4096 \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
[tab] slurm/jobs/mihir/horeka/coinrun/train_lam_single_gpu.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --time=00:20:00
#SBATCH --partition=dev_accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/lam/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/lam/%x_%j.log
#SBATCH --job-name=train_lam_1e-4
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/lam/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --save_ckpt \
    --image_height=64 \
    --image_width=64 \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=10 \
    --log_checkpoint_interval=100 \
    --log \
    --name=coinrun-lam-dev-$slurm_job_id \
    --tags lam coinrun dev \
    --entity instant-uv \
    --project jafar \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --num_steps 200 \
    --data_dir $array_records_dir_train
[tab] slurm/dev/mihir/horeka/chunked_ablation/train_dynamics_chunked_dataset copy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=00:30:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/maskgit/dynamics-cotraining/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/maskgit/dynamics-cotraining/%x_%j.log
#SBATCH --job-name=train_dynamics_maskgit_8_node_chunked
#SBATCH --requeue
#SBATCH --reservation=llmtum
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi



# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/holiday/maskgit/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401

env | grep SLURM

srun python train_dynamics.py \
    --num_steps 1000 \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=160 \
    --init_lr=0 \
    --dyna_type=maskgit \
    --num_latent_actions=100 \
    --max_lr=8e-5 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=dynamics-maskgit-8-node-chunked-data-$slurm_job_id \
    --tags dynamics maskgit 8-node chunked \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
[tab] slurm/dev/mihir/horeka/chunked_ablation/train_dynamics_unchunked_dataset.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=00:30:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/maskgit/dynamics-cotraining/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/maskgit/dynamics-cotraining/%x_%j.log
#SBATCH --job-name=train_dynamics_maskgit_8_node_chunked
#SBATCH --requeue
#SBATCH --reservation=llmtum
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi



# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/holiday/maskgit/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

tokenizer_ckpt_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/train_tokenizer_1e-4/3412401

env | grep SLURM

srun python train_dynamics.py \
    --num_steps 1000 \
    --warmup_steps 0 \
    --wsd_decay_steps 0 \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=160 \
    --init_lr=0 \
    --dyna_type=maskgit \
    --num_latent_actions=100 \
    --max_lr=8e-5 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=dynamics-maskgit-8-node-chunked-data-$slurm_job_id \
    --tags dynamics maskgit 8-node chunked \
    --entity instant-uv \
    --project jafar \
    --tokenizer_checkpoint=$tokenizer_ckpt_dir \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
[tab] slurm/jobs/mihir/horeka/lam/minecraft/train_lam_minecraft_8node.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=8
#SBATCH --ntasks-per-node=4
#SBATCH --time=48:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/lam/train_lam_minecraft_8node_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/lam/train_lam_minecraft_8node_%j.log
#SBATCH --job-name=train_lam_minecraft_8node
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords_chunked

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/lam/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $slurm_job_id \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=160 \
    --init_lr=0 \
    --max_lr=1e-4 \
    --log_image_interval=1000 \
    --log \
    --log_checkpoint_interval=1000 \
    --name=lam-minecraft-8-node-$slurm_job_id \
    --tags lam minecraft 8-node \
    --entity instant-uv \
    --project jafar \
    --num_latents=100 \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid