Repository: huggingface/dataset-viewer

Issue:
Do not support datasets that require manual download
Currently, datasets that require manual download show a weird error message due to a TypeError. See: https://huggingface.co/datasets/timit_asr/discussions/2
```
expected str, bytes or os.PathLike object, not NoneType
```

I think we should not support theses datasets, as we already do with private datasets.

EDIT: TODO:
- [x] Raise `DatasetManualDownloadError` from `config-parquet-and-info`
  - [x] #1309
- [x] Raise `DatasetManualDownloadError` from `config-split-names-from-streaming`
  - [x] #1315
- [ ] Propagate the error codes to the aggregator steps (config -> dataset, in this case) so that we can show the appropriate message on the Hub


Discussion:

Resolving patch:
diff --git a/libs/libcommon/src/libcommon/exceptions.py b/libs/libcommon/src/libcommon/exceptions.py
--- a/libs/libcommon/src/libcommon/exceptions.py
+++ b/libs/libcommon/src/libcommon/exceptions.py
@@ -76,6 +76,7 @@ def as_response(self) -> ErrorResponse:
     "ConfigNamesError",
     "DatasetInBlockListError",
     "DatasetInfoHubRequestError",
+    "DatasetManualDownloadError",
     "DatasetModuleNotInstalledError",
     "DatasetNotFoundError",
     "DatasetRevisionEmptyError",
@@ -158,6 +159,13 @@ def __init__(self, message: str, cause: Optional[BaseException] = None):
         )
 
 
+class DatasetManualDownloadError(CacheableError):
+    """Raised when the dataset requires manual download."""
+
+    def __init__(self, message: str, cause: Optional[BaseException] = None):
+        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "DatasetManualDownloadError", cause, True)
+
+
 class DatasetModuleNotInstalledError(CacheableError):
     """Raised when the dataset tries to import a module that is not installed."""
 
diff --git a/services/worker/src/worker/job_runners/config/parquet_and_info.py b/services/worker/src/worker/job_runners/config/parquet_and_info.py
--- a/services/worker/src/worker/job_runners/config/parquet_and_info.py
+++ b/services/worker/src/worker/job_runners/config/parquet_and_info.py
@@ -17,7 +17,7 @@
 import numpy as np
 import requests
 from datasets import DownloadConfig, get_dataset_config_info, load_dataset_builder
-from datasets.builder import DatasetBuilder
+from datasets.builder import DatasetBuilder, ManualDownloadError
 from datasets.data_files import EmptyDatasetError as _EmptyDatasetError
 from datasets.download import StreamingDownloadManager
 from datasets.utils.file_utils import (
@@ -43,6 +43,7 @@
 from libcommon.exceptions import (
     ConfigNamesError,
     DatasetInBlockListError,
+    DatasetManualDownloadError,
     DatasetNotFoundError,
     DatasetRevisionNotFoundError,
     DatasetTooBigFromDatasetsError,
@@ -290,6 +291,59 @@ def raise_if_too_big_from_datasets(
         )
 
 
+def raise_if_requires_manual_download(
+    dataset: str,
+    config: str,
+    hf_endpoint: str,
+    hf_token: Optional[str],
+    revision: str,
+) -> None:
+    """
+    Raise an error if the dataset requires manual download.
+
+    Args:
+        dataset (`str`):
+            A namespace (user or an organization) and a repo name separated
+            by a `/`.
+        config (`str`):
+            Dataset configuration name.
+        hf_endpoint (`str`):
+            The Hub endpoint (for example: "https://huggingface.co").
+        hf_token (`str`, *optional*):
+            An app authentication token with read access to all the datasets.
+        revision (`str`):
+            The git revision (e.g. "main" or sha) of the dataset.
+
+    Returns:
+        `None`
+
+    Raises:
+        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError):
+            If the datasets.config.HF_ENDPOINT is not set to the expected value.
+        [`libcommon.exceptions.DatasetManualDownloadError`]:
+            If the dataset requires manual download.
+    """
+    if datasets.config.HF_ENDPOINT != hf_endpoint:
+        raise ValueError(
+            f"Invalid datasets.config.HF_ENDPOINT value: '{datasets.config.HF_ENDPOINT}'. Please set it to:"
+            f" '{hf_endpoint}'."
+        )
+    builder = load_dataset_builder(
+        dataset,
+        name=config,
+        revision=revision,
+        use_auth_token=hf_token,
+    )
+    try:
+        builder._check_manual_download(
+            StreamingDownloadManager(
+                base_path=builder.base_path, download_config=DownloadConfig(use_auth_token=hf_token)
+            )
+        )
+    except ManualDownloadError as err:
+        raise DatasetManualDownloadError(f"{dataset=} requires manual download.", cause=err) from err
+
+
 def raise_if_not_supported(
     dataset: str,
     config: str,
@@ -336,6 +390,8 @@ def raise_if_not_supported(
           any other error when asking access
         - [`libcommon.exceptions.DatasetInBlockListError`]
           If the dataset is in the list of blocked datasets.
+        - [`libcommon.exceptions.DatasetManualDownloadError`]:
+          If the dataset requires manual download.
         - [`libcommon.exceptions.DatasetRevisionNotFoundError`]
           If the revision does not exist or cannot be accessed using the token.
         - [`libcommon.exceptions.DatasetTooBigFromDatasetsError`]
@@ -351,6 +407,13 @@ def raise_if_not_supported(
     dataset_info = get_dataset_info_or_raise(
         dataset=dataset, hf_endpoint=hf_endpoint, hf_token=hf_token, revision=revision
     )
+    raise_if_requires_manual_download(
+        dataset=dataset,
+        config=config,
+        hf_endpoint=hf_endpoint,
+        hf_token=hf_token,
+        revision=revision,
+    )
     if dataset in supported_datasets:
         return
     raise_if_too_big_from_datasets(
@@ -624,6 +687,8 @@ def compute_config_parquet_and_info_response(
             If the previous step gave an error.
         - [`libcommon.exceptions.DatasetInBlockListError`]
           If the dataset is in the list of blocked datasets.
+        - [`libcommon.exceptions.DatasetManualDownloadError`]:
+          If the dataset requires manual download.
         - [`libcommon.exceptions.DatasetRevisionNotFoundError`]
           If the revision does not exist or cannot be accessed using the token.
         - [`libcommon.exceptions.DatasetTooBigFromDatasetsError`]


Test patch:
diff --git a/services/worker/tests/fixtures/files.py b/services/worker/tests/fixtures/files.py
--- a/services/worker/tests/fixtures/files.py
+++ b/services/worker/tests/fixtures/files.py
@@ -147,3 +147,38 @@ def dataset_script_with_two_configs_path(tmp_path_factory: pytest.TempPathFactor
     with open(path, "w", newline="") as f:
         f.write(DATASET_SCRIPT_WITH_TWO_CONFIGS)
     return path
+
+
+DATASET_SCRIPT_WITH_MANUAL_DOWNLOAD = """
+import os
+
+import datasets
+from datasets import DatasetInfo, BuilderConfig, Features, Split, SplitGenerator, Value
+
+
+class DummyDatasetManualDownload(datasets.GeneratorBasedBuilder):
+
+    @property
+    def manual_download_instructions(self):
+        return "To use DummyDatasetManualDownload you have to download it manually."
+
+    def _info(self) -> DatasetInfo:
+        return DatasetInfo(features=Features({"text": Value("string")}))
+
+    def _split_generators(self, dl_manager):
+        return [
+            SplitGenerator(Split.TRAIN, gen_kwargs={"text": self.config.name}),
+        ]
+
+    def _generate_examples(self, text, **kwargs):
+        for i in range(1000):
+            yield i, {"text": text}
+"""
+
+
+@pytest.fixture(scope="session")
+def dataset_script_with_manual_download_path(tmp_path_factory: pytest.TempPathFactory) -> str:
+    path = str(tmp_path_factory.mktemp("data") / "{dataset_name}.py")
+    with open(path, "w", newline="") as f:
+        f.write(DATASET_SCRIPT_WITH_MANUAL_DOWNLOAD)
+    return path
diff --git a/services/worker/tests/fixtures/hub.py b/services/worker/tests/fixtures/hub.py
--- a/services/worker/tests/fixtures/hub.py
+++ b/services/worker/tests/fixtures/hub.py
@@ -242,6 +242,13 @@ def hub_public_legacy_configs(dataset_script_with_two_configs_path: str) -> Iter
     delete_hub_dataset_repo(repo_id=repo_id)
 
 
+@pytest.fixture(scope="session")
+def hub_public_manual_download(dataset_script_with_manual_download_path: str) -> Iterator[str]:
+    repo_id = create_hub_dataset_repo(prefix="manual_download", file_paths=[dataset_script_with_manual_download_path])
+    yield repo_id
+    delete_hub_dataset_repo(repo_id=repo_id)
+
+
 @pytest.fixture(scope="session")
 def hub_public_spawning_opt_in_out(datasets: Mapping[str, Dataset]) -> Iterator[str]:
     repo_id = create_hub_dataset_repo(prefix="spawning_opt_in_out", dataset=datasets["spawning_opt_in_out"])
diff --git a/services/worker/tests/job_runners/config/test_parquet_and_info.py b/services/worker/tests/job_runners/config/test_parquet_and_info.py
--- a/services/worker/tests/job_runners/config/test_parquet_and_info.py
+++ b/services/worker/tests/job_runners/config/test_parquet_and_info.py
@@ -17,6 +17,7 @@
 from libcommon.exceptions import (
     CustomError,
     DatasetInBlockListError,
+    DatasetManualDownloadError,
     DatasetTooBigFromDatasetsError,
     DatasetTooBigFromHubError,
     DatasetWithTooBigExternalFilesError,
@@ -35,6 +36,7 @@
     parse_repo_filename,
     raise_if_blocked,
     raise_if_not_supported,
+    raise_if_requires_manual_download,
     raise_if_too_big_from_datasets,
     raise_if_too_big_from_external_data_files,
     raise_if_too_big_from_hub,
@@ -230,6 +232,17 @@ def test_raise_if_blocked(dataset: str, blocked: List[str], raises: bool) -> Non
         raise_if_blocked(dataset=dataset, blocked_datasets=blocked)
 
 
+def test_raise_if_requires_manual_download(hub_public_manual_download: str, app_config: AppConfig) -> None:
+    with pytest.raises(DatasetManualDownloadError):
+        raise_if_requires_manual_download(
+            hub_public_manual_download,
+            "default",
+            hf_endpoint=app_config.common.hf_endpoint,
+            hf_token=app_config.common.hf_token,
+            revision="main",
+        )
+
+
 @pytest.mark.parametrize(
     "name,raises",
     [("public", False), ("big", True)],