p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] sample.py (python)

    # --- Sample + evaluate video ---
    # B, S, H, W, _ = video_batch_BSHWC.shape
    # N = math.ceil(H / args.patch_size) * math.ceil(W / args.patch_size)
    # for block in genie.dynamics.transformer.blocks:
    #     block.spatial_attention.init_cache((B * S, 1, args.dyna_dim), dtype=args.dtype)
    #     block.temporal_attention.init_cache((B * (N + 1), 1, args.dyna_dim), dtype=args.dtype)
[content] Untitled-1 (plaintext)
with jax.profiler.trace("/tmp/jax-trace", create_perfetto_link=True):
  # Run the operations to be profiled
  key = jax.random.key(0)
  x = jax.random.normal(key, (5000, 5000))
  y = x @ x
  y.block_until_ready()
[tab] experiments/dynamics_grain_full_restore.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=8
#SBATCH --time=12:00:00
#SBATCH --cpus-per-task=1
#SBATCH --gres=gpu:8
#SBATCH --mem=100GB

source .venv/bin/activate

data_dir="$PWD/data_arrayrecord/dummy"
ckpt_dir="$PWD/checkpoints/maskgit_dynamics_openai_grain_tok_restore"

export XLA_FLAGS=--xla_gpu_autotune_level=0
srun python train_dynamics.py \
    --batch_size 12 \
    --restore_ckpt \
    --save_ckpt \
    --log_checkpoint_interval 5 \
    --ckpt_dir $ckpt_dir \
    --num_steps 300000 \
    --warmup_steps 10000 \
    --seed 0 \
    --init_lr=0.0000866 \
    --max_lr=0.0000866 \
    --data_dir $data_dir
[tab] experiments/dynamics_grain_tok_restore.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=8
#SBATCH --time=12:00:00
#SBATCH --cpus-per-task=1
#SBATCH --gres=gpu:8
#SBATCH --mem=100GB

source .venv/bin/activate

data_dir="$PWD/data_arrayrecord/dummy"
ckpt_dir="$PWD/checkpoints/causal_dynamics_openai_grain_tok_restore"
tokenizer_ckpt_dir="$PWD/checkpoints/tokenizer_openai_grain_checkpointing"

export XLA_FLAGS=--xla_gpu_autotune_level=0
export PYTHONUNBUFFERED=1
srun python train_dynamics.py \
    --dyna_type 'causal' \
    --save_ckpt \
    --log_checkpoint_interval 5 \
    --batch_size 12 \
    --tokenizer_checkpoint $tokenizer_ckpt_dir \
    --ckpt_dir $ckpt_dir \
    --num_steps 300000 \
    --warmup_steps 10000 \
    --seed 0 \
    --init_lr=0.0000866 \
    --max_lr=0.0000866 \
    --data_dir $data_dir