GitHub issue #359: ArrowBasedBuilder _prepare_split parse_schema breaks on nested structures
State: closed | Created: 2020-07-08T23:24:05Z

I tried using the Json dataloader to load some JSON lines files. but get an exception in the parse_schema function.

```
---------------------------------------------------------------------------

ValueError                                Traceback (most recent call last)

<ipython-input-23-9aecfbee53bd> in <module>
     55 from nlp import load_dataset
     56 
---> 57 ds = load_dataset("../text2struct/model/dataset_builder.py", data_files=rel_datafiles)
     58 
     59 

~/.virtualenvs/inv-text2struct/lib/python3.6/site-packages/nlp/load.py in load_dataset(path, name, version, data_dir, data_files, split, cache_dir, download_config, download_mode, ignore_verifications, save_infos, **config_kwargs)
    522         download_mode=download_mode,
    523         ignore_verifications=ignore_verifications,
--> 524         save_infos=save_infos,
    525     )
    526 

~/.virtualenvs/inv-text2struct/lib/python3.6/site-packages/nlp/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, save_infos, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    430                 verify_infos = not save_infos and not ignore_verifications
    431                 self._download_and_prepare(
--> 432                     dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    433                 )
    434                 # Sync info

~/.virtualenvs/inv-text2struct/lib/python3.6/site-packages/nlp/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    481             try:
    482                 # Prepare split will record examples associated to the split
--> 483                 self._prepare_split(split_generator, **prepare_split_kwargs)
    484             except OSError:
    485                 raise OSError("Cannot find data file. " + (self.manual_download_instructions or ""))

~/.virtualenvs/inv-text2struct/lib/python3.6/site-packages/nlp/builder.py in _prepare_split(self, split_generator)
    736                     schema_dict[field.name] = Value(str(field.type))
    737 
--> 738         parse_schema(writer.schema, features)
    739         self.info.features = Features(features)
    740 

~/.virtualenvs/inv-text2struct/lib/python3.6/site-packages/nlp/builder.py in parse_schema(schema, schema_dict)
    734                     parse_schema(field.type.value_type, schema_dict[field.name])
    735                 else:
--> 736                     schema_dict[field.name] = Value(str(field.type))
    737 
    738         parse_schema(writer.schema, features)

<string> in __init__(self, dtype, id, _type)

~/.virtualenvs/inv-text2struct/lib/python3.6/site-packages/nlp/features.py in __post_init__(self)
     55 
     56     def __post_init__(self):
---> 57         self.pa_type = string_to_arrow(self.dtype)
     58 
     59     def __call__(self):

~/.virtualenvs/inv-text2struct/lib/python3.6/site-packages/nlp/features.py in string_to_arrow(type_str)
     32         if str(type_str + "_") not in pa.__dict__:
     33             raise ValueError(
---> 34                 f"Neither {type_str} nor {type_str + '_'} seems to be a pyarrow data type. "
     35                 f"Please make sure to use a correct data type, see: "
     36                 f"https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions"

ValueError: Neither list<item: string> nor list<item: string>_ seems to be a pyarrow data type. Please make sure to use a correct data type, see: https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions
```

If I create the dataset imperatively, using a pyarrow table, the dataset is created correctly. If I override the `_prepare_split` method to avoid calling the validate schema, the dataset can load as well.