Loading dataset with streaming: '_io.BufferedReader' object has no attribute 'loc'
Opened by milmin (closed)

## Describe the bug
Loading a datasets with `load_dataset` and `streaming=True` returns `AttributeError: '_io.BufferedReader' object has no attribute 'loc'`. Notice that loading with `streaming=False` works fine.

In the following steps we load parquet files but the same happens with pickle files. The problem seems to come from `fsspec` lib, I put in the environment info also `s3fs` and `fsspec` versions since I'm loading from an s3 bucket.

## Steps to reproduce the bug
```python
from datasets import load_dataset
# path is the path to parquet files
data_files = {"train": path + "meta_train.parquet.gzip", "test": path + "meta_test.parquet.gzip"}
dataset = load_dataset("parquet", data_files=data_files, streaming=True)
```

## Expected results
A dataset object `datasets.dataset_dict.DatasetDict`

## Actual results
```
AttributeError                            Traceback (most recent call last)
<command-562086> in <module>
     11 
     12 data_files = {"train": path + "meta_train.parquet.gzip", "test": path + "meta_test.parquet.gzip"}
---> 13 dataset = load_dataset("parquet", data_files=data_files, streaming=True)

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, keep_in_memory, save_infos, revision, use_auth_token, task, streaming, **config_kwargs)
   1679     if streaming:
   1680         extend_dataset_builder_for_streaming(builder_instance, use_auth_token=use_auth_token)
-> 1681         return builder_instance.as_streaming_dataset(
   1682             split=split,
   1683             use_auth_token=use_auth_token,

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/builder.py in as_streaming_dataset(self, split, base_path, use_auth_token)
    904         )
    905         self._check_manual_download(dl_manager)
--> 906         splits_generators = {sg.name: sg for sg in self._split_generators(dl_manager)}
    907         # By default, return all splits
    908         if split is None:

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/packaged_modules/parquet/parquet.py in _split_generators(self, dl_manager)
     30         if not self.config.data_files:
     31             raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
---> 32         data_files = dl_manager.download_and_extract(self.config.data_files)
     33         if isinstance(data_files, (str, list, tuple)):
     34             files = data_files

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/utils/streaming_download_manager.py in download_and_extract(self, url_or_urls)
    798 
    799     def download_and_extract(self, url_or_urls):
--> 800         return self.extract(self.download(url_or_urls))
    801 
    802     def iter_archive(self, urlpath_or_buf: Union[str, io.BufferedReader]) -> Iterable[Tuple]:

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/utils/streaming_download_manager.py in extract(self, path_or_paths)
    776 
    777     def extract(self, path_or_paths):
--> 778         urlpaths = map_nested(self._extract, path_or_paths, map_tuple=True)
    779         return urlpaths
    780 

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/utils/py_utils.py in map_nested(function, data_struct, dict_only, map_list, map_tuple, map_numpy, num_proc, types, disable_tqdm, desc)
    312         num_proc = 1
    313     if num_proc <= 1 or len(iterable) <= num_proc:
--> 314         mapped = [
    315             _single_map_nested((function, obj, types, None, True, None))
    316             for obj in logging.tqdm(iterable, disable=disable_tqdm, desc=desc)

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/utils/py_utils.py in <listcomp>(.0)
    313     if num_proc <= 1 or len(iterable) <= num_proc:
    314         mapped = [
--> 315             _single_map_nested((function, obj, types, None, True, None))
    316             for obj in logging.tqdm(iterable, disable=disable_tqdm, desc=desc)
    317         ]

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/utils/py_utils.py in _single_map_nested(args)
    267         return {k: _single_map_nested((function, v, types, None, True, None)) for k, v in pbar}
    268     else:
--> 269         mapped = [_single_map_nested((function, v, types, None, True, None)) for v in pbar]
    270         if isinstance(data_struct, list):
    271             return mapped

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/utils/py_utils.py in <listcomp>(.0)
    267         return {k: _single_map_nested((function, v, types, None, True, None)) for k, v in pbar}
    268     else:
--> 269         mapped = [_single_map_nested((function, v, types, None, True, None)) for v in pbar]
    270         if isinstance(data_struct, list):
    271             return mapped

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/utils/py_utils.py in _single_map_nested(args)
    249     # Singleton first to spare some computation
    250     if not isinstance(data_struct, dict) and not isinstance(data_struct, types):
--> 251         return function(data_struct)
    252 
    253     # Reduce logging to keep things readable in multiprocessing with tqdm

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/utils/streaming_download_manager.py in _extract(self, urlpath)
    781     def _extract(self, urlpath: str) -> str:
    782         urlpath = str(urlpath)
--> 783         protocol = _get_extraction_protocol(urlpath, use_auth_token=self.download_config.use_auth_token)
    784         if protocol is None:
    785             # no extraction

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/utils/streaming_download_manager.py in _get_extraction_protocol(urlpath, use_auth_token)
    371         urlpath, kwargs = urlpath, {}
    372     with fsspec.open(urlpath, **kwargs) as f:
--> 373         return _get_extraction_protocol_with_magic_number(f)
    374 
    375 

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/datasets/utils/streaming_download_manager.py in _get_extraction_protocol_with_magic_number(f)
    335 def _get_extraction_protocol_with_magic_number(f) -> Optional[str]:
    336     """read the magic number from a file-like object and return the compression protocol"""
--> 337     prev_loc = f.loc
    338     magic_number = f.read(MAGIC_NUMBER_MAX_LENGTH)
    339     f.seek(prev_loc)

/local_disk0/.ephemeral_nfs/envs/pythonEnv-a7e72260-221c-472b-85f4-bec801aee66d/lib/python3.8/site-packages/fsspec/implementations/local.py in __getattr__(self, item)
    337 
    338     def __getattr__(self, item):
--> 339         return getattr(self.f, item)
    340 
    341     def __enter__(self):

AttributeError: '_io.BufferedReader' object has no attribute 'loc'
```
## Environment info
- `datasets` version: 2.1.0
- Platform: Linux-5.4.0-1071-aws-x86_64-with-glibc2.29
- Python version: 3.8.10
- PyArrow version: 8.0.0
- Pandas version: 1.4.2
- `fsspec` version: 2021.08.1
- `s3fs` version: 2021.08.1

Comments: