GitHub issue #794: self.options cannot be converted to a Python object for pickling
State: closed | Created: 2020-11-03T09:27:34Z

Hi,

Currently I am trying to load csv file with customized read_options. And the latest master seems broken if we pass the ReadOptions object.

Here is a code snippet
```python
from datasets import load_dataset
from pyarrow.csv import ReadOptions
load_dataset("csv", data_files=["out.csv"], read_options=ReadOptions(block_size=16*1024*1024))
```
error is `self.options cannot be converted to a Python object for pickling`
Would you mind to take a look? Thanks!

```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-28-ab83fec2ded4> in <module>
----> 1 load_dataset("csv", data_files=["out.csv"], read_options=ReadOptions(block_size=16*1024*1024))

/tmp/datasets/src/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    602         hash=hash,
    603         features=features,
--> 604         **config_kwargs,
    605     )
    606 

/tmp/datasets/src/datasets/builder.py in __init__(self, cache_dir, name, hash, features, **config_kwargs)
    162             name,
    163             custom_features=features,
--> 164             **config_kwargs,
    165         )
    166 

/tmp/datasets/src/datasets/builder.py in _create_builder_config(self, name, custom_features, **config_kwargs)
    281                 )
    282             else:
--> 283                 suffix = Hasher.hash(config_kwargs_to_add_to_suffix)
    284 
    285         if builder_config.data_files is not None:

/tmp/datasets/src/datasets/fingerprint.py in hash(cls, value)
     51             return cls.dispatch[type(value)](cls, value)
     52         else:
---> 53             return cls.hash_default(value)
     54 
     55     def update(self, value):

/tmp/datasets/src/datasets/fingerprint.py in hash_default(cls, value)
     44     @classmethod
     45     def hash_default(cls, value):
---> 46         return cls.hash_bytes(dumps(value))
     47 
     48     @classmethod

/tmp/datasets/src/datasets/utils/py_utils.py in dumps(obj)
    365     file = StringIO()
    366     with _no_cache_fields(obj):
--> 367         dump(obj, file)
    368     return file.getvalue()
    369 

/tmp/datasets/src/datasets/utils/py_utils.py in dump(obj, file)
    337 def dump(obj, file):
    338     """pickle an object to a file"""
--> 339     Pickler(file, recurse=True).dump(obj)
    340     return
    341 

~/.local/lib/python3.6/site-packages/dill/_dill.py in dump(self, obj)
    444             raise PicklingError(msg)
    445         else:
--> 446             StockPickler.dump(self, obj)
    447         stack.clear()  # clear record of 'recursion-sensitive' pickled objects
    448         return

/usr/lib/python3.6/pickle.py in dump(self, obj)
    407         if self.proto >= 4:
    408             self.framer.start_framing()
--> 409         self.save(obj)
    410         self.write(STOP)
    411         self.framer.end_framing()

/usr/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
    474         f = self.dispatch.get(t)
    475         if f is not None:
--> 476             f(self, obj) # Call unbound method with explicit self
    477             return
    478 

~/.local/lib/python3.6/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    931             # we only care about session the first pass thru
    932             pickler._session = False
--> 933         StockPickler.save_dict(pickler, obj)
    934         log.info("# D2")
    935     return

/usr/lib/python3.6/pickle.py in save_dict(self, obj)
    819 
    820         self.memoize(obj)
--> 821         self._batch_setitems(obj.items())
    822 
    823     dispatch[dict] = save_dict

/usr/lib/python3.6/pickle.py in _batch_setitems(self, items)
    850                 k, v = tmp[0]
    851                 save(k)
--> 852                 save(v)
    853                 write(SETITEM)
    854             # else tmp is empty, and we're done

/usr/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
    494             reduce = getattr(obj, "__reduce_ex__", None)
    495             if reduce is not None:
--> 496                 rv = reduce(self.proto)
    497             else:
    498                 reduce = getattr(obj, "__reduce__", None)

~/.local/lib/python3.6/site-packages/pyarrow/_csv.cpython-36m-x86_64-linux-gnu.so in pyarrow._csv.ReadOptions.__reduce_cython__()

TypeError: self.options cannot be converted to a Python object for pickling
```