[tab] utils/dataloader.py (python)
import jax
import numpy as np
import grain
from typing import Any
import pickle


<<<<<<< HEAD
class ProcessEpisodeAndSlice(grain.transforms.RandomMap):
    """
    A Grain Transformation that combines parsing, slicing, and normalizing.
=======
# --- TensorFlow function for processing: slicing, normalization ---
def _tf_process_episode(episode_tensor, seq_len, image_h, image_w, image_c, seed):
    """
    Processes a raw episode tensor in TensorFlow.
    Takes a full episode, extracts a random sequence, and normalizes it.
    Args:
        episode_tensor: A TensorFlow tensor representing a full video episode.
                        Expected shape: (dynamic_length, image_h, image_w, image_c)
                        Expected dtype: e.g., tf.uint8 (raw pixel values)
        seq_len: The desired length of the sub-sequence to extract.
        image_h: The height of each frame.
        image_w: The width of each frame.
        image_c: The number of channels in each frame.
        seed: The seed for the random number generator.
    Returns:
        A TensorFlow tensor representing the processed video sequence.
        Shape: (seq_len, image_h, image_w, image_c)
        Dtype: tf.float32 (normalized pixel values)
>>>>>>> main
    """

    def __init__(self, seq_len: int, image_h: int, image_w: int, image_c: int):
        """Initializes the transformation with processing parameters."""
        self.seq_len = seq_len
        self.image_h = image_h
        self.image_w = image_w
        self.image_c = image_c

<<<<<<< HEAD
    def random_map(self, element: dict, rng: np.random.Generator) -> Any:
        """
        Processes a single raw episode from the data source.
=======
    start_idx = tf.random.uniform(
        shape=(), minval=0, maxval=max_start_idx + 1, dtype=tf.int32, seed=seed
    )
>>>>>>> main

        Args:
            element: A dictionary representing one record from the DataSource.
                     Expected to contain 'raw_video' (bytes) and 'sequence_length' (int)
            rng: A per-record random number generator provided by the Grain sampler.

        Returns:
            A processed video sequence as a NumPy array with shape
            (seq_len, height, width, channels) and dtype float32.
        """
        assert isinstance(element, bytes)
        element = pickle.loads(element)
        
        video_shape = (
            element["sequence_length"],
            self.image_h,
            self.image_w,
            self.image_c,
        )
        episode_tensor = np.frombuffer(element["raw_video"], dtype=np.uint8)
        episode_tensor = episode_tensor.reshape(video_shape)

        current_episode_len = episode_tensor.shape[0]
        if current_episode_len < self.seq_len:
             raise ValueError(f"An episode has length {current_episode_len}, which is "
                              f"shorter than the requested sequence length {self.seq_len}.")
        
        max_start_idx = current_episode_len - self.seq_len
        
        start_idx = rng.integers(0, max_start_idx + 1)

        seq = episode_tensor[start_idx : start_idx + self.seq_len]

        processed_sequence = seq.astype(np.float32) / 255.0

        return processed_sequence


def get_dataloader(
    array_record_paths: list[str],
    seq_len: int,
    global_batch_size: int,
    image_h: int,
    image_w: int,
    image_c: int,
    num_workers: int = 1,
    prefetch_buffer_size: int = 1,
    seed: int = 42,
):
    """
    Creates a data loading pipeline using Grain.
    """
    if not array_record_paths:
        raise ValueError("array_record_paths list cannot be empty.")

    num_processes = jax.process_count()

    if global_batch_size % num_processes != 0:
        raise ValueError(
            f"Global batch size {global_batch_size} must be divisible by "
            f"the number of JAX processes {num_processes} for proper sharding."
        )
    per_process_batch_size = global_batch_size // num_processes

    source = grain.sources.ArrayRecordDataSource(array_record_paths)
    
    sampler = grain.samplers.IndexSampler(
        num_records=len(source),
        shard_options=grain.sharding.ShardByJaxProcess(drop_remainder=True),
        shuffle=True,
        num_epochs=100,
        seed=seed,
    )

    operations = [
        ProcessEpisodeAndSlice(
            seq_len=seq_len, image_h=image_h, image_w=image_w, image_c=image_c
        ),
        grain.transforms.Batch(batch_size=per_process_batch_size, drop_remainder=True),
    ]

    read_options = grain.ReadOptions(
        prefetch_buffer_size=prefetch_buffer_size,
        num_threads=1,
    )
<<<<<<< HEAD
    dataloader = grain.DataLoader(
        data_source=source,
        sampler=sampler,
        operations=operations,
        worker_count=num_workers,
        worker_buffer_size=1,
        read_options=read_options,
=======
    dataset = dataset.map(parse_fn, num_parallel_calls=num_parallel_calls)

    tf_process_fn = functools.partial(
        _tf_process_episode,
        seq_len=seq_len,
        image_h=image_h,
        image_w=image_w,
        image_c=image_c,
        seed=seed,
>>>>>>> main
    )

    return iter(dataloader)