Repository: huggingface/datasets
File: src/datasets/packaged_modules/arrow/arrow.py (Python)

Code before:
                    with open(file, "rb") as f:
                        try:
                            reader = pa.ipc.open_stream(f)
                        except pa.lib.ArrowInvalid:
                            reader = pa.ipc.open_file(f)
                    self.info.features = datasets.Features.from_arrow_schema(reader.schema)
                    break
            splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
        return splits

    def _cast_table(self, pa_table: pa.Table) -> pa.Table:
        if self.info.features is not None:
            # more expensive cast to support nested features with keys in a different order
            # allows str <-> int/float or str to Audio for example
            pa_table = table_cast(pa_table, self.info.features.arrow_schema)
        return pa_table

    def _generate_tables(self, files):
        for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
            with open(file, "rb") as f:
                try:
                    try:
                        batches = pa.ipc.open_stream(f)
                    except pa.lib.ArrowInvalid:
                        reader = pa.ipc.open_file(f)
                        batches = [reader.get_batch(i) for i in range(reader.num_record_batches)]
                    for batch_idx, record_batch in enumerate(batches):
                        pa_table = pa.Table.from_batches([record_batch])
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                        yield f"{file_idx}_{batch_idx}", self._cast_table(pa_table)
                except ValueError as e:
                    logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
                    raise


Review comment (performance):
(nit) save some memory by using a generator and not have all the batches in memory at the same time

```suggestion
                        batches = (reader.get_batch(i) for i in range(reader.num_record_batches))
```

Code after:
                    with open(file, "rb") as f:
                        try:
                            reader = pa.ipc.open_stream(f)
                        except pa.lib.ArrowInvalid:
                            reader = pa.ipc.open_file(f)
                    self.info.features = datasets.Features.from_arrow_schema(reader.schema)
                    break
            splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
        return splits

    def _cast_table(self, pa_table: pa.Table) -> pa.Table:
        if self.info.features is not None:
            # more expensive cast to support nested features with keys in a different order
            # allows str <-> int/float or str to Audio for example
            pa_table = table_cast(pa_table, self.info.features.arrow_schema)
        return pa_table

    def _generate_tables(self, files):
        for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
            with open(file, "rb") as f:
                try:
                    try:
                        batches = pa.ipc.open_stream(f)
                    except pa.lib.ArrowInvalid:
                        reader = pa.ipc.open_file(f)
                        batches = (reader.get_batch(i) for i in range(reader.num_record_batches))
                    for batch_idx, record_batch in enumerate(batches):
                        pa_table = pa.Table.from_batches([record_batch])
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                        yield f"{file_idx}_{batch_idx}", self._cast_table(pa_table)
                except ValueError as e:
                    logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
                    raise