Issue: Get an error "OverflowError: Python int too large to convert to C long" when loading a large dataset

Comment:
I am afraid int32 is not the reason for this error.

I have submitted a commit to use int64 for all ints in the dataset:
https://huggingface.co/datasets/liwu/MNBVC/commit/857ac00d9eab96a6708ad6a82bd9001686042a9e

and I have updated my env to the latest datasets release:
Copy-and-paste the text below in your GitHub issue.

- `datasets` version: 2.13.1
- Platform: macOS-13.2.1-arm64-arm-64bit
- Python version: 3.11.2
- Huggingface_hub version: 0.13.4
- PyArrow version: 11.0.0
- Pandas version: 1.5.3

But the error still exist

```
Downloading and preparing dataset mnbvc/news_peoples_daily to /Users/silver/.cache/huggingface/datasets/liwu___mnbvc/news_peoples_daily/0.0.1/ee380f6309fe9b8b0d1fb14d77118f132444f22c8c4b28bf5c1645312688e051...
Downloading data files: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 9070.40it/s]
Extracting data files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12/12 [00:00<00:00, 2697.16it/s]
---------------------------------------------------------------------------
OverflowError                             Traceback (most recent call last)
File ~/git/venv/lib/python3.11/site-packages/datasets/builder.py:1647, in GeneratorBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, split_info, check_duplicate_keys, job_id)
   1646 example = self.info.features.encode_example(record) if self.info.features is not None else record
-> 1647 writer.write(example, key)
   1648 num_examples_progress_update += 1

File ~/git/venv/lib/python3.11/site-packages/datasets/arrow_writer.py:490, in ArrowWriter.write(self, example, key, writer_batch_size)
    488     self.hkey_record = []
--> 490 self.write_examples_on_file()

File ~/git/venv/lib/python3.11/site-packages/datasets/arrow_writer.py:448, in ArrowWriter.write_examples_on_file(self)
    444         batch_examples[col] = [
    445             row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col]
    446             for row in self.current_examples
    447         ]
--> 448 self.write_batch(batch_examples=batch_examples)
    449 self.current_examples = []

File ~/git/venv/lib/python3.11/site-packages/datasets/arrow_writer.py:553, in ArrowWriter.write_batch(self, batch_examples, writer_batch_size)
    552 typed_sequence = OptimizedTypedSequence(col_values, type=col_type, try_type=col_try_type, col=col)
--> 553 arrays.append(pa.array(typed_sequence))
    554 inferred_features[col] = typed_sequence.get_inferred_type()

File ~/git/venv/lib/python3.11/site-packages/pyarrow/array.pxi:236, in pyarrow.lib.array()

File ~/git/venv/lib/python3.11/site-packages/pyarrow/array.pxi:110, in pyarrow.lib._handle_arrow_array_protocol()

File ~/git/venv/lib/python3.11/site-packages/datasets/arrow_writer.py:189, in TypedSequence.__arrow_array__(self, type)
    188     trying_cast_to_python_objects = True
--> 189     out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
    190 # use smaller integer precisions if possible

File ~/git/venv/lib/python3.11/site-packages/pyarrow/array.pxi:320, in pyarrow.lib.array()

File ~/git/venv/lib/python3.11/site-packages/pyarrow/array.pxi:39, in pyarrow.lib._sequence_to_array()

File ~/git/venv/lib/python3.11/site-packages/pyarrow/error.pxi:144, in pyarrow.lib.pyarrow_internal_check_status()

OverflowError: Python int too large to convert to C long

During handling of the above exception, another exception occurred:

OverflowError                             Traceback (most recent call last)
File ~/git/venv/lib/python3.11/site-packages/datasets/builder.py:1656, in GeneratorBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, split_info, check_duplicate_keys, job_id)
   1655 num_shards = shard_id + 1
-> 1656 num_examples, num_bytes = writer.finalize()
   1657 writer.close()

File ~/git/venv/lib/python3.11/site-packages/datasets/arrow_writer.py:584, in ArrowWriter.finalize(self, close_stream)
    583     self.hkey_record = []
--> 584 self.write_examples_on_file()
    585 # If schema is known, infer features even if no examples were written

File ~/git/venv/lib/python3.11/site-packages/datasets/arrow_writer.py:448, in ArrowWriter.write_examples_on_file(self)
    444         batch_examples[col] = [
    445             row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col]
    446             for row in self.current_examples
    447         ]
--> 448 self.write_batch(batch_examples=batch_examples)
    449 self.current_examples = []

File ~/git/venv/lib/python3.11/site-packages/datasets/arrow_writer.py:553, in ArrowWriter.write_batch(self, batch_examples, writer_batch_size)
    552 typed_sequence = OptimizedTypedSequence(col_values, type=col_type, try_type=col_try_type, col=col)
--> 553 arrays.append(pa.array(typed_sequence))
    554 inferred_features[col] = typed_sequence.get_inferred_type()

File ~/git/venv/lib/python3.11/site-packages/pyarrow/array.pxi:236, in pyarrow.lib.array()

File ~/git/venv/lib/python3.11/site-packages/pyarrow/array.pxi:110, in pyarrow.lib._handle_arrow_array_protocol()

File ~/git/venv/lib/python3.11/site-packages/datasets/arrow_writer.py:189, in TypedSequence.__arrow_array__(self, type)
    188     trying_cast_to_python_objects = True
--> 189     out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
    190 # use smaller integer precisions if possible

File ~/git/venv/lib/python3.11/site-packages/pyarrow/array.pxi:320, in pyarrow.lib.array()

File ~/git/venv/lib/python3.11/site-packages/pyarrow/array.pxi:39, in pyarrow.lib._sequence_to_array()

File ~/git/venv/lib/python3.11/site-packages/pyarrow/error.pxi:144, in pyarrow.lib.pyarrow_internal_check_status()

OverflowError: Python int too large to convert to C long

The above exception was the direct cause of the following exception:

DatasetGenerationError                    Traceback (most recent call last)
Cell In[2], line 1
----> 1 dataset = load_dataset("liwu/MNBVC", 'news_peoples_daily', split='train')

File ~/git/venv/lib/python3.11/site-packages/datasets/load.py:1809, in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, verification_mode, ignore_verifications, keep_in_memory, save_infos, revision, use_auth_token, task, streaming, num_proc, storage_options, **config_kwargs)
   1806 try_from_hf_gcs = path not in _PACKAGED_DATASETS_MODULES
   1808 # Download and prepare data
-> 1809 builder_instance.download_and_prepare(
   1810     download_config=download_config,
   1811     download_mode=download_mode,
   1812     verification_mode=verification_mode,
   1813     try_from_hf_gcs=try_from_hf_gcs,
   1814     num_proc=num_proc,
   1815     storage_options=storage_options,
   1816 )
   1818 # Build dataset for splits
   1819 keep_in_memory = (
   1820     keep_in_memory if keep_in_memory is not None else is_small_dataset(builder_instance.info.dataset_size)
   1821 )

File ~/git/venv/lib/python3.11/site-packages/datasets/builder.py:909, in DatasetBuilder.download_and_prepare(self, output_dir, download_config, download_mode, verification_mode, ignore_verifications, try_from_hf_gcs, dl_manager, base_path, use_auth_token, file_format, max_shard_size, num_proc, storage_options, **download_and_prepare_kwargs)
    907     if num_proc is not None:
    908         prepare_split_kwargs["num_proc"] = num_proc
--> 909     self._download_and_prepare(
    910         dl_manager=dl_manager,
    911         verification_mode=verification_mode,
    912         **prepare_split_kwargs,
    913         **download_and_prepare_kwargs,
    914     )
    915 # Sync info
    916 self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())

File ~/git/venv/lib/python3.11/site-packages/datasets/builder.py:1670, in GeneratorBasedBuilder._download_and_prepare(self, dl_manager, verification_mode, **prepare_splits_kwargs)
   1669 def _download_and_prepare(self, dl_manager, verification_mode, **prepare_splits_kwargs):
-> 1670     super()._download_and_prepare(
   1671         dl_manager,
   1672         verification_mode,
   1673         check_duplicate_keys=verification_mode == VerificationMode.BASIC_CHECKS
   1674         or verification_mode == VerificationMode.ALL_CHECKS,
   1675         **prepare_splits_kwargs,
   1676     )

File ~/git/venv/lib/python3.11/site-packages/datasets/builder.py:1004, in DatasetBuilder._download_and_prepare(self, dl_manager, verification_mode, **prepare_split_kwargs)
   1000 split_dict.add(split_generator.split_info)
   1002 try:
   1003     # Prepare split will record examples associated to the split
-> 1004     self._prepare_split(split_generator, **prepare_split_kwargs)
   1005 except OSError as e:
   1006     raise OSError(
   1007         "Cannot find data file. "
   1008         + (self.manual_download_instructions or "")
   1009         + "\nOriginal error:\n"
   1010         + str(e)
   1011     ) from None

File ~/git/venv/lib/python3.11/site-packages/datasets/builder.py:1508, in GeneratorBasedBuilder._prepare_split(self, split_generator, check_duplicate_keys, file_format, num_proc, max_shard_size)
   1506 job_id = 0
   1507 with pbar:
-> 1508     for job_id, done, content in self._prepare_split_single(
   1509         gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args
   1510     ):
   1511         if done:
   1512             result = content

File ~/git/venv/lib/python3.11/site-packages/datasets/builder.py:1665, in GeneratorBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, split_info, check_duplicate_keys, job_id)
   1663     if isinstance(e, SchemaInferenceError) and e.__context__ is not None:
   1664         e = e.__context__
-> 1665     raise DatasetGenerationError("An error occurred while generating the dataset") from e
   1667 yield job_id, True, (total_num_examples, total_num_bytes, writer._features, num_shards, shard_lengths)

DatasetGenerationError: An error occurred while generating the dataset
```

Besides, it works fine when I am using streamed dataset.