Repository: huggingface/datasets
File: src/datasets/iterable_dataset.py (Python)

Code before:
        self.with_indices = with_indices
        self.input_columns = input_columns
        self.fn_kwargs = fn_kwargs or {}
        self.formatting = formatting
        # sanity checks
        if formatting and formatting.format_type == "arrow":
            # batch_size should match for iter_arrow
            if not isinstance(ex_iterable, RebatchedArrowExamplesIterable):
                raise ValueError(
                    "The Arrow-formatted MappedExamplesIterable has underlying iterable"
                    f"that is a {type(ex_iterable).__name__} instead of a RebatchedArrowExamplesIterable."
                )
            elif ex_iterable.batch_size != (batch_size if batched else 1):
                raise ValueError(
                    f"The Arrow-formatted MappedExamplesIterable has batch_size={batch_size if batched else 1} which is"
                    f"different from {ex_iterable.batch_size=} from its underlying iterable."
                )

    @property
    def iter_arrow(self):
        if self.formatting and self.formatting.format_type == "arrow":
            return self._iter_arrow

    @property
    def is_typed(self):
        return self.ex_iterable.is_typed

    def _init_state_dict(self) -> dict:
        self._state_dict = {
            "ex_iterable": self.ex_iterable._init_state_dict(),
            "previous_state": None,
            "num_examples_since_previous_state": 0,
            "previous_state_example_idx": 0,
        }
        return self._state_dict

    def __iter__(self):
        if self.formatting and self.formatting.format_type == "arrow":
            formatter = PythonFormatter()
            for key, pa_table in self._iter_arrow(max_chunksize=1):
                yield key, formatter.format_row(pa_table)
        else:
            yield from self._iter()

    def _iter(self):
        current_idx = self._state_dict["previous_state_example_idx"] if self._state_dict else 0
        if self._state_dict and self._state_dict["previous_state"]:
            self.ex_iterable.load_state_dict(self._state_dict["previous_state"])
            num_examples_to_skip = self._state_dict["num_examples_since_previous_state"]
        else:
            num_examples_to_skip = 0

Review comment (suggestion):
we can't know the output of a mapped iterable, so it should be False no ?
```suggestion
        return False
```

Code after:
        self.with_indices = with_indices
        self.input_columns = input_columns
        self.fn_kwargs = fn_kwargs or {}
        self.formatting = formatting
        # sanity checks
        if formatting and formatting.format_type == "arrow":
            # batch_size should match for iter_arrow
            if not isinstance(ex_iterable, RebatchedArrowExamplesIterable):
                raise ValueError(
                    "The Arrow-formatted MappedExamplesIterable has underlying iterable"
                    f"that is a {type(ex_iterable).__name__} instead of a RebatchedArrowExamplesIterable."
                )
            elif ex_iterable.batch_size != (batch_size if batched else 1):
                raise ValueError(
                    f"The Arrow-formatted MappedExamplesIterable has batch_size={batch_size if batched else 1} which is"
                    f"different from {ex_iterable.batch_size=} from its underlying iterable."
                )

    @property
    def iter_arrow(self):
        if self.formatting and self.formatting.format_type == "arrow":
            return self._iter_arrow

    @property
    def is_typed(self):
        return False

    def _init_state_dict(self) -> dict:
        self._state_dict = {
            "ex_iterable": self.ex_iterable._init_state_dict(),
            "previous_state": None,
            "num_examples_since_previous_state": 0,
            "previous_state_example_idx": 0,
        }
        return self._state_dict

    def __iter__(self):
        if self.formatting and self.formatting.format_type == "arrow":
            formatter = PythonFormatter()
            for key, pa_table in self._iter_arrow(max_chunksize=1):
                yield key, formatter.format_row(pa_table)
        else:
            yield from self._iter()

    def _iter(self):
        current_idx = self._state_dict["previous_state_example_idx"] if self._state_dict else 0
        if self._state_dict and self._state_dict["previous_state"]:
            self.ex_iterable.load_state_dict(self._state_dict["previous_state"])
            num_examples_to_skip = self._state_dict["num_examples_since_previous_state"]
        else:
            num_examples_to_skip = 0