[content] sample.py (python)
    # Additional parameters
    darkness_threshold: float = 0.0


args = tyro.cli(Args)

if __name__ == "__main__":
    """
    Dimension keys:
        B: batch size
        T: number of input (conditioning) frames
        N: number of patches per frame
        S: sequence length
        H: height
        W: width
        E: B * (S - 1)
    """
    jax.distributed.initialize()

    rng = jax.random.key(args.seed)

    # --- Load Genie checkpoint ---
    rngs = nnx.Rngs(rng)
    genie = Genie(
        # Tokenizer
        in_dim=args.image_channels,
        tokenizer_dim=args.tokenizer_dim,
        tokenizer_ffn_dim=args.tokenizer_ffn_dim,
        latent_patch_dim=args.latent_patch_dim,
        num_patch_latents=args.num_patch_latents,
        patch_size=args.patch_size,
        tokenizer_num_blocks=args.tokenizer_num_blocks,
        tokenizer_num_heads=args.tokenizer_num_heads,
        # LAM
        lam_dim=args.lam_dim,
        lam_ffn_dim=args.lam_ffn_dim,
        latent_action_dim=args.latent_action_dim,
        num_latent_actions=args.num_latent_actions,
        lam_patch_size=args.lam_patch_size,
        lam_num_blocks=args.lam_num_blocks,
        lam_num_heads=args.lam_num_heads,
        lam_co_train=False,
        # Dynamics
        dyna_type=args.dyna_type,
        dyna_dim=args.dyna_dim,
        dyna_ffn_dim=args.dyna_ffn_dim,
        dyna_num_blocks=args.dyna_num_blocks,
        dyna_num_heads=args.dyna_num_heads,
        param_dtype=args.param_dtype,
        dtype=args.dtype,
        use_flash_attention=args.use_flash_attention,
        # FIXME (f.srambical): implement spatiotemporal KV caching and set decode=True
        decode=False,
        rngs=rngs,
    )

    handler_registry = ocp.handlers.DefaultCheckpointHandlerRegistry()
    handler_registry.add(
        "model_state", ocp.args.PyTreeSave, ocp.handlers.PyTreeCheckpointHandler
    )
    handler_registry.add(
        "model_state", ocp.args.PyTreeRestore, ocp.handlers.PyTreeCheckpointHandler
    )
    checkpoint_options = ocp.CheckpointManagerOptions(
        step_format_fixed_length=6,
    )
    checkpoint_manager = ocp.CheckpointManager(
        args.checkpoint,
        options=checkpoint_options,
        handler_registry=handler_registry,
    )

    dummy_tx = optax.adamw(
        learning_rate=optax.linear_schedule(0.0001, 0.0001, 10000),
        b1=0.9,
        b2=0.9,
        weight_decay=1e-4,
        mu_dtype=args.dtype,
    )
    dummy_optimizer = nnx.Optimizer(genie, dummy_tx)

    abstract_optimizer = nnx.eval_shape(lambda: dummy_optimizer)
    abstract_optimizer_state = nnx.state(abstract_optimizer)
    restored = checkpoint_manager.restore(
        checkpoint_manager.latest_step(),
        args=ocp.args.Composite(
            model_state=ocp.args.PyTreeRestore(abstract_optimizer_state),  # type: ignore
        ),
    )
    restored_optimizer_state = restored["model_state"]
    nnx.update(dummy_optimizer, restored_optimizer_state)

    # --- Define sampling function ---
    def _sampling_fn(model: Genie, batch: dict) -> jax.Array:
        """Runs Genie.sample with pre-defined generation hyper-parameters."""
        if args.dyna_type == "maskgit":
            return model.sample(
                batch,
                args.seq_len,
                args.maskgit_steps,
                args.temperature,
                args.sample_argmax,
            )
        elif args.dyna_type == "causal":
            return model.sample_causal(
                batch,
                args.seq_len,
                args.temperature,
                args.sample_argmax,
            )
        else:
            raise ValueError(f"Invalid dynamics type: {args.dyna_type}")

    # --- Define autoregressive sampling loop ---
    def _autoreg_sample(genie, rng, video_batch_BSHWC, action_batch_E):
        input_video_BTHWC = video_batch_BSHWC[:, : args.start_frame]
        rng, _rng = jax.random.split(rng)
        batch = dict(videos=input_video_BTHWC, latent_actions=action_batch_E, rng=_rng)
        generated_vid_BSHWC = _sampling_fn(genie, batch)
        return generated_vid_BSHWC

    # --- Get video + latent actions ---
    array_record_files = [
        os.path.join(args.data_dir, x)
        for x in os.listdir(args.data_dir)
        if x.endswith(".array_record")
    ]
    dataloader = get_dataloader(
        array_record_files,
        args.seq_len,
        args.batch_size,
        args.image_height,
        args.image_width,
        args.image_channels,
        # We don't use workers in order to avoid grain shutdown issues (https://github.com/google/grain/issues/398)
        num_workers=0,
        prefetch_buffer_size=1,
        seed=args.seed,
        darkness_threshold=args.darkness_threshold,