Make `ted_talks_iwslt` dataset streamable
State: closed | Author: mariosasko (COLLABORATOR)
Labels: name: dataset contribution


Comments: Thanks for adding this @mariosasko! It worked for me when running it with a local data file, however, when using the file on Google Drive I get the following error:
```Python
ds = load_dataset("./ted_talks_iwslt","eu_ca_2014", streaming=True, split="train", use_auth_token=True)
next(iter(ds))
```
```
---------------------------------------------------------------------------
ClientResponseError                       Traceback (most recent call last)
File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/implementations/http.py:383, in HTTPFileSystem._info(self, url, **kwargs)
    381 try:
    382     info.update(
--> 383         await _file_info(
    384             url,
    385             size_policy=policy,
    386             session=session,
    387             **self.kwargs,
    388             **kwargs,
    389         )
    390     )
    391     if info.get("size") is not None:

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/implementations/http.py:734, in _file_info(url, session, size_policy, **kwargs)
    733 async with r:
--> 734     r.raise_for_status()
    736     # TODO:
    737     #  recognise lack of 'Accept-Ranges',
    738     #                 or 'Accept-Ranges': 'none' (not 'bytes')
    739     #  to mean streaming only, no random access => return None

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/aiohttp/client_reqrep.py:1004, in ClientResponse.raise_for_status(self)
   1003 self.release()
-> 1004 raise ClientResponseError(
   1005     self.request_info,
   1006     self.history,
   1007     status=self.status,
   1008     message=self.reason,
   1009     headers=self.headers,
   1010 )

ClientResponseError: 403, message='Forbidden', url=URL('https://drive.google.com/u/0/uc?id=1Cz1Un9p8Xn9IpEMMrg2kXSDt0dnjxc4z&export=download&confirm=1RJz')

The above exception was the direct cause of the following exception:

FileNotFoundError                         Traceback (most recent call last)
Input In [9], in <module>
      1 iterable = iter(ds)
      2 for i in range(10):
----> 3     item = next(iterable)
      4     print(item['text'][:10], item['meta'])

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/datasets/iterable_dataset.py:341, in IterableDataset.__iter__(self)
    340 def __iter__(self):
--> 341     for key, example in self._iter():
    342         if self.features:
    343             # we encode the example for ClassLabel feature types for example
    344             encoded_example = self.features.encode_example(example)

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/datasets/iterable_dataset.py:338, in IterableDataset._iter(self)
    336 else:
    337     ex_iterable = self._ex_iterable
--> 338 yield from ex_iterable

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/datasets/iterable_dataset.py:78, in ExamplesIterable.__iter__(self)
     77 def __iter__(self):
---> 78     for key, example in self.generate_examples_fn(**self.kwargs):
     79         yield key, example

File ~/.cache/huggingface/modules/datasets_modules/datasets/lm_en_ted_talks_iwslt/756148758e86e64a350f9b320744a2bd5ed5cff74f7df620763a2b5e1a45e6c6/lm_en_ted_talks_iwslt.py:118, in TedTalksIWSLT._generate_examples(self, files)
    116 for _LANG in _LANG_CODES:
    117     source_file_path = _YEAR_FOLDER[year] + "/ted_" + _LANG + _YEAR[year] + ".zip"
--> 118     for path, file in files:
    119         if path.endswith(source_file_path):
    120             source_talks, _ = parse_zip_file(path, file.read())

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/datasets/utils/streaming_download_manager.py:596, in StreamingDownloadManager.iter_archive(self, urlpath_or_buf)
    594     yield from _iter_archive(urlpath_or_buf)
    595 else:
--> 596     with xopen(urlpath_or_buf, "rb", use_auth_token=self.download_config.use_auth_token) as f:
    597         yield from _iter_archive(f)

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/datasets/utils/streaming_download_manager.py:296, in xopen(file, mode, use_auth_token, *args, **kwargs)
    294     new_kwargs = {}
    295 kwargs = {**kwargs, **new_kwargs}
--> 296 file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open()
    297 _add_retries_to_file_obj_read_method(file_obj)
    298 return file_obj

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/core.py:140, in OpenFile.open(self)
    132 def open(self):
    133     """Materialise this as a real open file without context
    134 
    135     The file should be explicitly closed to avoid enclosed file
   (...)
    138     been deleted; but a with-context is better style.
    139     """
--> 140     out = self.__enter__()
    141     closer = out.close
    142     fobjects = self.fobjects.copy()[:-1]

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/core.py:103, in OpenFile.__enter__(self)
    100 def __enter__(self):
    101     mode = self.mode.replace("t", "").replace("b", "") + "b"
--> 103     f = self.fs.open(self.path, mode=mode)
    105     self.fobjects = [f]
    107     if self.compression is not None:

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/spec.py:1009, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)
   1007 else:
   1008     ac = kwargs.pop("autocommit", not self._intrans)
-> 1009     f = self._open(
   1010         path,
   1011         mode=mode,
   1012         block_size=block_size,
   1013         autocommit=ac,
   1014         cache_options=cache_options,
   1015         **kwargs,
   1016     )
   1017     if compression is not None:
   1018         from fsspec.compression import compr

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/implementations/http.py:343, in HTTPFileSystem._open(self, path, mode, block_size, autocommit, cache_type, cache_options, size, **kwargs)
    341 kw["asynchronous"] = self.asynchronous
    342 kw.update(kwargs)
--> 343 size = size or self.info(path, **kwargs)["size"]
    344 session = sync(self.loop, self.set_session)
    345 if block_size and size:

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/asyn.py:91, in sync_wrapper.<locals>.wrapper(*args, **kwargs)
     88 @functools.wraps(func)
     89 def wrapper(*args, **kwargs):
     90     self = obj or args[0]
---> 91     return sync(self.loop, func, *args, **kwargs)

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/asyn.py:71, in sync(loop, func, timeout, *args, **kwargs)
     69     raise FSTimeoutError from return_result
     70 elif isinstance(return_result, BaseException):
---> 71     raise return_result
     72 else:
     73     return return_result

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/asyn.py:25, in _runner(event, coro, result, timeout)
     23     coro = asyncio.wait_for(coro, timeout=timeout)
     24 try:
---> 25     result[0] = await coro
     26 except Exception as ex:
     27     result[0] = ex

File ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/implementations/http.py:396, in HTTPFileSystem._info(self, url, **kwargs)
    393     except Exception as exc:
    394         if policy == "get":
    395             # If get failed, then raise a FileNotFoundError
--> 396             raise FileNotFoundError(url) from exc
    397         logger.debug(str(exc))
    399 return {"name": url, "size": None, **info, "type": "file"}

FileNotFoundError: https://drive.google.com/u/0/uc?id=1Cz1Un9p8Xn9IpEMMrg2kXSDt0dnjxc4z&export=download&confirm=1RJz
```

Thanks @mariosasko.

To make this dataset streamable, we should first host the data on the Hub instead of current Google Drive. Do you know if their license allows to do so? 

This dataset is licensed under [cc-by-nc-4.0](https://creativecommons.org/licenses/by-nc/4.0/), so I think it should be