Repository: huggingface/dataset-viewer

Issue:
RowsPostProcessingError in the viewer of several datasets
We find a `RowsPostProcessingError` in the viewer of several datasets:
```
Server error while post-processing the split rows. Please report the issue.

Error code:   RowsPostProcessingError
```
- https://huggingface.co/datasets/multi_woz_v22
- https://huggingface.co/datasets/qasper


Discussion:
It seems like the issue is that we don't support a Sequence of dicts adequately:

> A [datasets.Sequence](https://huggingface.co/docs/datasets/v2.5.2/en/package_reference/main_classes#datasets.Sequence) with a internal dictionary feature will be automatically converted into a dictionary of lists. This behavior is implemented to have a compatilbity layer with the TensorFlow Datasets library but may be un-wanted in some cases. If you don’t want this behavior, you can use a python list instead of the [datasets.Sequence](https://huggingface.co/docs/datasets/v2.5.2/en/package_reference/main_classes#datasets.Sequence).

https://huggingface.co/docs/datasets/v2.5.2/en/package_reference/main_classes#datasets.Features



Resolving patch:
diff --git a/services/worker/src/worker/features.py b/services/worker/src/worker/features.py
--- a/services/worker/src/worker/features.py
+++ b/services/worker/src/worker/features.py
@@ -147,41 +147,49 @@ def get_cell_value(
             for (idx, subCell) in enumerate(cell)
         ]
     elif isinstance(fieldType, Sequence):
-        # sequence value can be a list or a dict, see
+        # if the internal feature of the Sequence is a dict, then the value will automatically
+        # be converted into a dictionary of lists. See
         # https://huggingface.co/docs/datasets/v2.5.1/en/package_reference/main_classes#datasets.Features
-        if type(cell) == list:
-            if fieldType.length >= 0 and len(cell) != fieldType.length:
-                raise TypeError("the cell length should be the same as the Sequence length.")
-            return [
-                get_cell_value(
-                    dataset,
-                    config,
-                    split,
-                    row_idx,
-                    subCell,
-                    featureName,
-                    fieldType.feature,
-                    assets_base_url,
-                    json_path + [idx] if json_path else [idx],
-                )
-                for (idx, subCell) in enumerate(cell)
-            ]
-        elif type(cell) == dict:
+        if type(fieldType.feature) == dict:
+            if type(cell) != dict or any(type(k) != list for k in cell.values()):
+                raise TypeError("The value of a Sequence of dicts should be a dictionary of lists.")
             return {
-                key: get_cell_value(
-                    dataset,
-                    config,
-                    split,
-                    row_idx,
-                    subCell,
-                    featureName,
-                    fieldType.feature[key],
-                    assets_base_url,
-                    json_path + [key] if json_path else [key],
-                )
+                key: [
+                    get_cell_value(
+                        dataset,
+                        config,
+                        split,
+                        row_idx,
+                        subCellItem,
+                        featureName,
+                        fieldType.feature[key],
+                        assets_base_url,
+                        json_path + [key, idx] if json_path else [key, idx],
+                    )
+                    for (idx, subCellItem) in enumerate(subCell)
+                ]
                 for (key, subCell) in cell.items()
             }
-        raise TypeError("Sequence cell must be a list or a dict.")
+        # else: it must be a list
+        if type(cell) != list:
+            raise TypeError("Sequence cell must be a list or a dict.")
+        if fieldType.length >= 0 and len(cell) != fieldType.length:
+            raise TypeError("the cell length should be the same as the Sequence length.")
+        return [
+            get_cell_value(
+                dataset,
+                config,
+                split,
+                row_idx,
+                subCell,
+                featureName,
+                fieldType.feature,
+                assets_base_url,
+                json_path + [idx] if json_path else [idx],
+            )
+            for (idx, subCell) in enumerate(cell)
+        ]
+
     elif isinstance(fieldType, dict):
         if type(cell) != dict:
             raise TypeError("dict cell must be a dict.")
diff --git a/services/worker/src/worker/responses/first_rows.py b/services/worker/src/worker/responses/first_rows.py
--- a/services/worker/src/worker/responses/first_rows.py
+++ b/services/worker/src/worker/responses/first_rows.py
@@ -4,7 +4,7 @@
 import itertools
 import logging
 import sys
-from typing import Any, Dict, List, Optional, TypedDict
+from typing import Any, Dict, List, Optional, TypedDict, Union
 
 from datasets import (
     Dataset,
@@ -63,14 +63,14 @@ def get_rows(
     split: str,
     streaming: bool,
     rows_max_number: int,
-    hf_token: Optional[str] = None,
+    use_auth_token: Union[bool, str, None] = False,
 ) -> List[Row]:
     ds = load_dataset(
         dataset,
         name=config,
         split=split,
         streaming=streaming,
-        use_auth_token=hf_token,
+        use_auth_token=use_auth_token,
     )
     if streaming:
         if not isinstance(ds, IterableDataset):
@@ -288,6 +288,7 @@ def get_first_rows_response(
     </Tip>
     """
     logger.info(f"get first-rows for dataset={dataset} config={config} split={split}")
+    use_auth_token: Union[bool, str, None] = hf_token if hf_token is not None else False
     if rows_max_bytes is None:
         rows_max_bytes = DEFAULT_ROWS_MAX_BYTES
     if rows_max_number is None:
@@ -313,7 +314,7 @@ def get_first_rows_response(
         info = get_dataset_config_info(
             path=dataset,
             config_name=config,
-            use_auth_token=hf_token,
+            use_auth_token=use_auth_token,
         )
     except Exception as err:
         raise InfoError("The info cannot be fetched for the dataset config.", cause=err) from err
@@ -325,7 +326,7 @@ def get_first_rows_response(
                 name=config,
                 split=split,
                 streaming=True,
-                use_auth_token=hf_token,
+                use_auth_token=use_auth_token,
             )
             if not isinstance(iterable_dataset, IterableDataset):
                 raise TypeError("load_dataset should return an IterableDataset")
@@ -339,7 +340,9 @@ def get_first_rows_response(
         features = info.features
     # get the rows
     try:
-        rows = get_rows(dataset, config, split, streaming=True, rows_max_number=rows_max_number, hf_token=hf_token)
+        rows = get_rows(
+            dataset, config, split, streaming=True, rows_max_number=rows_max_number, use_auth_token=use_auth_token
+        )
     except Exception as err:
         if max_size_fallback is None or info.size_in_bytes is None or info.size_in_bytes > max_size_fallback:
             raise StreamingRowsError(
@@ -353,7 +356,7 @@ def get_first_rows_response(
                 split,
                 streaming=False,
                 rows_max_number=rows_max_number,
-                hf_token=hf_token,
+                use_auth_token=use_auth_token,
             )
         except Exception as err:
             raise NormalRowsError(
diff --git a/services/worker/src/worker/responses/splits.py b/services/worker/src/worker/responses/splits.py
--- a/services/worker/src/worker/responses/splits.py
+++ b/services/worker/src/worker/responses/splits.py
@@ -2,7 +2,7 @@
 # Copyright 2022 The HuggingFace Authors.
 
 import logging
-from typing import Dict, List, Optional, TypedDict
+from typing import Dict, List, Optional, TypedDict, Union
 
 from datasets import (
     DatasetInfo,
@@ -34,12 +34,12 @@ class SplitsResponse(TypedDict):
     splits: List[SplitItem]
 
 
-def get_dataset_split_full_names(dataset: str, hf_token: Optional[str] = None) -> List[SplitFullName]:
+def get_dataset_split_full_names(dataset: str, use_auth_token: Union[bool, str, None] = False) -> List[SplitFullName]:
     logger.info(f"get dataset '{dataset}' split full names")
     return [
         {"dataset": dataset, "config": config, "split": split}
-        for config in get_dataset_config_names(dataset, use_auth_token=hf_token)
-        for split in get_dataset_split_names(dataset, config, use_auth_token=hf_token)
+        for config in get_dataset_config_names(dataset, use_auth_token=use_auth_token)
+        for split in get_dataset_split_names(dataset, config, use_auth_token=use_auth_token)
     ]
 
 
@@ -71,14 +71,15 @@ def get_splits_response(
     </Tip>
     """
     logger.info(f"get splits for dataset={dataset}")
+    use_auth_token: Union[bool, str, None] = hf_token if hf_token is not None else False
     # first try to get the dataset config info
     try:
-        HfApi(endpoint=hf_endpoint).dataset_info(dataset, token=hf_token)
+        HfApi(endpoint=hf_endpoint).dataset_info(dataset, use_auth_token=use_auth_token)
     except RepositoryNotFoundError as err:
         raise DatasetNotFoundError("The dataset does not exist on the Hub.") from err
     # get the list of splits
     try:
-        split_full_names = get_dataset_split_full_names(dataset, hf_token)
+        split_full_names = get_dataset_split_full_names(dataset, use_auth_token)
     except _EmptyDatasetError as err:
         raise EmptyDatasetError("The dataset is empty.", cause=err) from err
     except Exception as err:
@@ -95,7 +96,7 @@ def get_splits_response(
                 config_info[config] = get_dataset_config_info(
                     path=dataset,
                     config_name=config,
-                    use_auth_token=hf_token,
+                    use_auth_token=use_auth_token,
                 )
             info = config_info[config]
             num_bytes = info.splits[split].num_bytes if info.splits else None


Test patch:
diff --git a/services/worker/tests/fixtures/datasets.py b/services/worker/tests/fixtures/datasets.py
--- a/services/worker/tests/fixtures/datasets.py
+++ b/services/worker/tests/fixtures/datasets.py
@@ -123,4 +123,7 @@ def datasets() -> Dict[str, Dataset]:
             },
             {"a": Value(dtype="int64"), "b": [Image()], "c": {"ca": [Audio()]}},
         ),
+        "sequence_of_dicts": other(
+            [{"a": {"b": 0}}, {"a": {"b": 1}}], Sequence(feature={"a": {"b": Value(dtype="int64")}})
+        ),
     }
diff --git a/services/worker/tests/fixtures/hub.py b/services/worker/tests/fixtures/hub.py
--- a/services/worker/tests/fixtures/hub.py
+++ b/services/worker/tests/fixtures/hub.py
@@ -16,7 +16,7 @@
     REPO_TYPES_URL_PREFIXES,
     HfApi,
     HfFolder,
-    _raise_for_status,
+    hf_raise_for_status,
 )
 
 from ..utils import get_default_config_split
@@ -72,8 +72,6 @@ def update_repo_settings(
 
     organization, name = repo_id.split("/") if "/" in repo_id else (None, repo_id)
 
-    token, name = hf_api._validate_or_retrieve_token(token, name, function_name="update_repo_settings")
-
     if organization is None:
         namespace = hf_api.whoami(token)["name"]
     else:
@@ -96,7 +94,7 @@ def update_repo_settings(
         headers={"authorization": f"Bearer {token}"},
         json=json,
     )
-    _raise_for_status(r)
+    hf_raise_for_status(r)
     return r.json()
 
 
diff --git a/services/worker/tests/responses/test_first_rows.py b/services/worker/tests/responses/test_first_rows.py
--- a/services/worker/tests/responses/test_first_rows.py
+++ b/services/worker/tests/responses/test_first_rows.py
@@ -21,8 +21,8 @@
         ("private", True, None, None),
         ("empty", False, "EmptyDatasetError", "EmptyDatasetError"),
         ("does_not_exist", False, "DatasetNotFoundError", None),
-        ("gated", False, "SplitsNamesError", "FileNotFoundError"),
-        ("private", False, "SplitsNamesError", "FileNotFoundError"),
+        ("gated", False, "DatasetNotFoundError", None),
+        ("private", False, "DatasetNotFoundError", None),
     ],
 )
 def test_number_rows(
diff --git a/services/worker/tests/responses/test_splits.py b/services/worker/tests/responses/test_splits.py
--- a/services/worker/tests/responses/test_splits.py
+++ b/services/worker/tests/responses/test_splits.py
@@ -19,8 +19,8 @@
         ("private", True, None, None),
         ("empty", False, "EmptyDatasetError", "EmptyDatasetError"),
         ("does_not_exist", False, "DatasetNotFoundError", None),
-        ("gated", False, "SplitsNamesError", "FileNotFoundError"),
-        ("private", False, "SplitsNamesError", "FileNotFoundError"),
+        ("gated", False, "DatasetNotFoundError", None),
+        ("private", False, "DatasetNotFoundError", None),
     ],
 )
 def test_get_splits_response_simple_csv(
diff --git a/services/worker/tests/test_features.py b/services/worker/tests/test_features.py
--- a/services/worker/tests/test_features.py
+++ b/services/worker/tests/test_features.py
@@ -273,6 +273,7 @@ def test_value(dataset_type, output_value, output_dtype, datasets) -> None:
             },
             {"a": Value(dtype="int64"), "b": [Image(decode=True, id=None)], "c": {"ca": [Audio()]}},
         ),
+        ("sequence_of_dicts", {"a": [{"b": 0}, {"b": 1}]}, "Sequence"),
     ],
 )
 def test_others(dataset_type: str, output_value: Any, output_type: Any, datasets: Dict[str, Dataset]) -> None: