Exceeded maximum rows when reading large files
State: closed | Author: zijwang (NONE)
Labels: name: bug

## Describe the bug
A clear and concise description of what the bug is.
When using `load_dataset` with json files, if the files are too large, there will be "Exceeded maximum rows" error.

## Steps to reproduce the bug
```python
dataset = load_dataset('json', data_files=data_files) # data files have 3M rows in a single file
```

## Expected results
No error

## Actual results
```
~/anaconda3/envs/python/lib/python3.9/site-packages/datasets/packaged_modules/json/json.py in _generate_tables(self, files)
    134                                 with open(file, encoding="utf-8") as f:
--> 135                                     dataset = json.load(f)
    136                             except json.JSONDecodeError:

~/anaconda3/envs/python/lib/python3.9/json/__init__.py in load(fp, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
    292     """
--> 293     return loads(fp.read(),
    294         cls=cls, object_hook=object_hook,

~/anaconda3/envs/python/lib/python3.9/json/__init__.py in loads(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
    345             parse_constant is None and object_pairs_hook is None and not kw):
--> 346         return _default_decoder.decode(s)
    347     if cls is None:

~/anaconda3/envs/python/lib/python3.9/json/decoder.py in decode(self, s, _w)
    339         if end != len(s):
--> 340             raise JSONDecodeError("Extra data", s, end)
    341         return obj

JSONDecodeError: Extra data: line 2 column 1 (char 20321)

During handling of the above exception, another exception occurred:

ArrowInvalid                              Traceback (most recent call last)
<ipython-input-20-ab3718a6482f> in <module>
----> 1 dataset = load_dataset('json', data_files=data_files)

~/anaconda3/envs/python/lib/python3.9/site-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, keep_in_memory, save_infos, script_version, use_auth_token, task, streaming, **config_kwargs)
    841 
    842     # Download and prepare data
--> 843     builder_instance.download_and_prepare(
    844         download_config=download_config,
    845         download_mode=download_mode,

~/anaconda3/envs/python/lib/python3.9/site-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, base_path, use_auth_token, **download_and_prepare_kwargs)
    606                             logger.warning("HF google storage unreachable. Downloading and preparing it from source")
    607                     if not downloaded_from_gcs:
--> 608                         self._download_and_prepare(
    609                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    610                         )

~/anaconda3/envs/python/lib/python3.9/site-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    684             try:
    685                 # Prepare split will record examples associated to the split
--> 686                 self._prepare_split(split_generator, **prepare_split_kwargs)
    687             except OSError as e:
    688                 raise OSError(

~/anaconda3/envs/python/lib/python3.9/site-packages/datasets/builder.py in _prepare_split(self, split_generator)
   1153         generator = self._generate_tables(**split_generator.gen_kwargs)
   1154         with ArrowWriter(features=self.info.features, path=fpath) as writer:
-> 1155             for key, table in utils.tqdm(
   1156                 generator, unit=" tables", leave=False, disable=bool(logging.get_verbosity() == logging.NOTSET)
   1157             ):


~/anaconda3/envs/python/lib/python3.9/site-packages/datasets/packaged_modules/json/json.py in _generate_tables(self, files)
    135                                     dataset = json.load(f)
    136                             except json.JSONDecodeError:
--> 137                                 raise e
    138                             raise ValueError(
    139                                 f"Not able to read records in the JSON file at {file}. "

~/anaconda3/envs/python/lib/python3.9/site-packages/datasets/packaged_modules/json/json.py in _generate_tables(self, files)
    114                             while True:
    115                                 try:
--> 116                                     pa_table = paj.read_json(
    117                                         BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size)
    118                                     )

~/anaconda3/envs/python/lib/python3.9/site-packages/pyarrow/_json.pyx in pyarrow._json.read_json()

~/anaconda3/envs/python/lib/python3.9/site-packages/pyarrow/error.pxi in pyarrow.lib.pyarrow_internal_check_status()

~/anaconda3/envs/python/lib/python3.9/site-packages/pyarrow/error.pxi in pyarrow.lib.check_status()

ArrowInvalid: Exceeded maximum rows
```

## Environment info
<!-- You can run the command `datasets-cli env` and copy-and-paste its output below. -->
- `datasets` version:
- Platform: Linux
- Python version: 3.9
- PyArrow version: 4.0.1


Comments: Hi @zijwang, thanks for reporting this issue.

You did not mention which `datasets` version you are using, but looking at the code in the stack trace, it seems you are using an old version.

Could you please update `datasets` (`pip install -U datasets`) and check if the problem persists?