raruidol (NONE) commented on 2020-09-30 17:44:01:
Thank you @lhoestq I have tried again with the new text reader and there is still some error. Depending on how do I load the data I have spotted two different crashes. When I try to  load the full-size corpus text file I get the following output:

```
Traceback (most recent call last):
  File "run_language_modeling.py", line 27, in <module>
    dataset = load_dataset('text', script_version='master', data_files=['corpora/cc_ca.txt', 'corpora/viquipedia_clean.txt'])['train']
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/datasets/builder.py", line 471, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/datasets/builder.py", line 548, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/datasets/builder.py", line 892, in _prepare_split
    for key, table in utils.tqdm(generator, unit=" tables", leave=False, disable=not_verbose):
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/tqdm/std.py", line 1130, in __iter__
    for obj in iterable:
  File "/home/jupyter-raruidol/.cache/huggingface/modules/datasets_modules/datasets/text/7c0f8af3178a6c694d530e71edd8f1ec1d7311867f8c29955898c29a0624adf5/text.py", line 68, in _generate_tables
    for j, df in enumerate(text_file_reader):
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/pandas/io/parsers.py", line 1161, in __next__
    return self.get_chunk()
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/pandas/io/parsers.py", line 1220, in get_chunk
    return self.read(nrows=size)
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/pandas/io/parsers.py", line 1186, in read
    ret = self._engine.read(nrows)
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/pandas/io/parsers.py", line 2145, in read
    data = self._reader.read(nrows)
  File "pandas/_libs/parsers.pyx", line 847, in pandas._libs.parsers.TextReader.read
  File "pandas/_libs/parsers.pyx", line 874, in pandas._libs.parsers.TextReader._read_low_memory
  File "pandas/_libs/parsers.pyx", line 918, in pandas._libs.parsers.TextReader._read_rows
  File "pandas/_libs/parsers.pyx", line 905, in pandas._libs.parsers.TextReader._tokenize_rows
  File "pandas/_libs/parsers.pyx", line 2042, in pandas._libs.parsers.raise_parser_error
pandas.errors.ParserError: Error tokenizing data. C error: Buffer overflow caught - possible malformed input file.
```

However, when loading the sharded version, the error is different:

```
Traceback (most recent call last):
  File "run_language_modeling.py", line 27, in <module>
    dataset = load_dataset('text', script_version='master', data_files=files)['train']
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/datasets/builder.py", line 471, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/datasets/builder.py", line 548, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/datasets/builder.py", line 892, in _prepare_split
    for key, table in utils.tqdm(generator, unit=" tables", leave=False, disable=not_verbose):
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/tqdm/std.py", line 1130, in __iter__
    for obj in iterable:
  File "/home/jupyter-raruidol/.cache/huggingface/modules/datasets_modules/datasets/text/7c0f8af3178a6c694d530e71edd8f1ec1d7311867f8c29955898c29a0624adf5/text.py", line 68, in _generate_tables
    for j, df in enumerate(text_file_reader):
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/pandas/io/parsers.py", line 1161, in __next__
    return self.get_chunk()
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/pandas/io/parsers.py", line 1220, in get_chunk
    return self.read(nrows=size)
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/pandas/io/parsers.py", line 1186, in read
    ret = self._engine.read(nrows)
  File "/home/jupyter-raruidol/DebatAnalyser/env/lib/python3.7/site-packages/pandas/io/parsers.py", line 2145, in read
    data = self._reader.read(nrows)
  File "pandas/_libs/parsers.pyx", line 847, in pandas._libs.parsers.TextReader.read
  File "pandas/_libs/parsers.pyx", line 874, in pandas._libs.parsers.TextReader._read_low_memory
  File "pandas/_libs/parsers.pyx", line 918, in pandas._libs.parsers.TextReader._read_rows
  File "pandas/_libs/parsers.pyx", line 905, in pandas._libs.parsers.TextReader._tokenize_rows
  File "pandas/_libs/parsers.pyx", line 2042, in pandas._libs.parsers.raise_parser_error
pandas.errors.ParserError: Error tokenizing data. C error: EOF inside string starting at row 254729
```