Loading 'wikitext' dataset fails
Author: itay1itzhak | Pull request: no | State: closed

Loading the 'wikitext' dataset fails with Attribute error:

Code to reproduce (From example notebook):

import nlp
wikitext_dataset = nlp.load_dataset('wikitext')


Error:
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-17-d5d9df94b13c> in <module>()
     11 
     12 # Load a dataset and print the first examples in the training set
---> 13 wikitext_dataset = nlp.load_dataset('wikitext')
     14 print(wikitext_dataset['train'][0])

6 frames
/usr/local/lib/python3.6/dist-packages/nlp/load.py in load_dataset(path, name, version, data_dir, data_files, split, cache_dir, download_config, download_mode, ignore_verifications, save_infos, **config_kwargs)
    518         download_mode=download_mode,
    519         ignore_verifications=ignore_verifications,
--> 520         save_infos=save_infos,
    521     )
    522 

/usr/local/lib/python3.6/dist-packages/nlp/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, save_infos, dl_manager, **download_and_prepare_kwargs)
    363                 verify_infos = not save_infos and not ignore_verifications
    364                 self._download_and_prepare(
--> 365                     dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    366                 )
    367                 # Sync info

/usr/local/lib/python3.6/dist-packages/nlp/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    416             try:
    417                 # Prepare split will record examples associated to the split
--> 418                 self._prepare_split(split_generator, **prepare_split_kwargs)
    419             except OSError:
    420                 raise OSError("Cannot find data file. " + (self.MANUAL_DOWNLOAD_INSTRUCTIONS or ""))

/usr/local/lib/python3.6/dist-packages/nlp/builder.py in _prepare_split(self, split_generator)
    594             example = self.info.features.encode_example(record)
    595             writer.write(example)
--> 596         num_examples, num_bytes = writer.finalize()
    597 
    598         assert num_examples == num_examples, f"Expected to write {split_info.num_examples} but wrote {num_examples}"

/usr/local/lib/python3.6/dist-packages/nlp/arrow_writer.py in finalize(self, close_stream)
    173     def finalize(self, close_stream=True):
    174         if self.pa_writer is not None:
--> 175             self.write_on_file()
    176             self.pa_writer.close()
    177         if close_stream:

/usr/local/lib/python3.6/dist-packages/nlp/arrow_writer.py in write_on_file(self)
    124             else:
    125                 # All good
--> 126                 self._write_array_on_file(pa_array)
    127             self.current_rows = []
    128 

/usr/local/lib/python3.6/dist-packages/nlp/arrow_writer.py in _write_array_on_file(self, pa_array)
     93     def _write_array_on_file(self, pa_array):
     94         """Write a PyArrow Array"""
---> 95         pa_batch = pa.RecordBatch.from_struct_array(pa_array)
     96         self._num_bytes += pa_array.nbytes
     97         self.pa_writer.write_batch(pa_batch)

AttributeError: type object 'pyarrow.lib.RecordBatch' has no attribute 'from_struct_array'