Repository: huggingface/dataset-viewer

Issue:
Regenerate dataset-info instead of loading it?
Currently, getting the rows with `/rows` requires a previous (internal) call to `/infos` to get the features (type of the columns). But sometimes the dataset-info.json file is missing, or not coherent with the dataset script (for example: https://huggingface.co/datasets/lhoestq/custom_squad/tree/main), while we are using `datasets.get_dataset_infos()`, which only loads the exported dataset-info.json files:

https://github.com/huggingface/datasets-preview-backend/blob/c2a78e7ce8e36cdf579fea805535fa9ef84a2027/src/datasets_preview_backend/queries/infos.py#L45


https://github.com/huggingface/datasets/blob/26ff41aa3a642e46489db9e95be1e9a8c4e64bea/src/datasets/inspect.py#L115


We might want to call `._info()` from the builder to get the info, and features, instead of relying on the dataset-info.json file.



Discussion:

Resolving patch:
diff --git a/src/datasets_preview_backend/queries/infos.py b/src/datasets_preview_backend/queries/infos.py
--- a/src/datasets_preview_backend/queries/infos.py
+++ b/src/datasets_preview_backend/queries/infos.py
@@ -1,7 +1,7 @@
 from dataclasses import asdict
 from typing import List, Optional, cast
 
-from datasets import get_dataset_infos
+from datasets import load_dataset_builder
 
 from datasets_preview_backend.cache import memoize  # type: ignore
 from datasets_preview_backend.config import CACHE_TTL_SECONDS, cache
@@ -42,12 +42,9 @@ def get_infos(dataset: str, config: Optional[str] = None, token: Optional[str] =
     # Note that we raise on the first error
     for config in configs:
         try:
-            dataset_info_dict = get_dataset_infos(dataset, use_auth_token=token)
-            if config not in dataset_info_dict:
-                # this is an error in the dataset-info.json
-                raise Exception("The config is not listed in the datasets infos")
-            info = asdict(dataset_info_dict[config])
-            if "splits" in info:
+            builder = load_dataset_builder(dataset, name=config, use_auth_token=token)
+            info = asdict(builder.info)
+            if "splits" in info and info["splits"] is not None:
                 info["splits"] = {split_name: split_info for split_name, split_info in info["splits"].items()}
         except FileNotFoundError as err:
             raise Status404Error("The config info could not be found.") from err
diff --git a/src/datasets_preview_backend/queries/rows.py b/src/datasets_preview_backend/queries/rows.py
--- a/src/datasets_preview_backend/queries/rows.py
+++ b/src/datasets_preview_backend/queries/rows.py
@@ -14,7 +14,7 @@
 from datasets_preview_backend.responses import CachedResponse
 from datasets_preview_backend.types import (
     ConfigsContent,
-    FeatureItem,
+    FeaturesItem,
     InfosContent,
     RowItem,
     RowsContent,
@@ -45,7 +45,7 @@ def get_rows(
     num_rows = EXTRACT_ROWS_LIMIT
 
     rowItems: List[RowItem] = []
-    featureItems: List[FeatureItem] = []
+    featuresItems: List[FeaturesItem] = []
 
     if config is not None and split is not None:
         try:
@@ -104,7 +104,7 @@ def get_rows(
         infoItem = infoItems[0]
         if "features" not in infoItem:
             raise Status400Error("a dataset config info should contain a 'features' property")
-        localFeaturesItems: List[FeatureItem] = [
+        localFeaturesItems: List[FeaturesItem] = [
             {"dataset": dataset, "config": config, "features": infoItem["features"]}
         ]
 
@@ -142,12 +142,12 @@ def get_rows(
                 raise Status400Error("rows could not be found")
             rows_content = cast(RowsContent, content)
             rowItems += rows_content["rows"]
-            for featureItem in rows_content["features"]:
+            for featuresItem in rows_content["features"]:
                 # there should be only one element. Anyway, let's loop
-                if featureItem not in featureItems:
-                    featureItems.append(featureItem)
+                if featuresItem not in featuresItems:
+                    featuresItems.append(featuresItem)
 
-    return {"features": featureItems, "rows": rowItems}
+    return {"features": featuresItems, "rows": rowItems}
 
 
 @memoize(cache, expire=CACHE_TTL_SECONDS)  # type:ignore
diff --git a/src/datasets_preview_backend/types.py b/src/datasets_preview_backend/types.py
--- a/src/datasets_preview_backend/types.py
+++ b/src/datasets_preview_backend/types.py
@@ -29,7 +29,7 @@ class RowItem(TypedDict):
     row: Any
 
 
-class FeatureItem(TypedDict):
+class FeaturesItem(TypedDict):
     dataset: str
     config: str
     features: Any
@@ -55,7 +55,7 @@ class SplitsContent(TypedDict):
 
 
 class RowsContent(TypedDict):
-    features: List[FeatureItem]
+    features: List[FeaturesItem]
     rows: List[RowItem]
 
 


Test patch:
diff --git a/tests/queries/test_infos.py b/tests/queries/test_infos.py
--- a/tests/queries/test_infos.py
+++ b/tests/queries/test_infos.py
@@ -34,6 +34,12 @@ def test_get_infos_no_config() -> None:
     assert len(infoItems) == 12
 
 
+def test_get_infos_no_dataset_info_file() -> None:
+    dataset = "lhoestq/custom_squad"
+    response = get_infos(dataset)
+    assert len(response["infos"]) == 1
+
+
 def test_script_error() -> None:
     # raises "ModuleNotFoundError: No module named 'datasets_modules.datasets.br-quad-2'"
     # which should be caught and raised as DatasetBuilderScriptError
diff --git a/tests/queries/test_rows.py b/tests/queries/test_rows.py
--- a/tests/queries/test_rows.py
+++ b/tests/queries/test_rows.py
@@ -37,11 +37,11 @@ def test_get_split_features() -> None:
     response = get_rows(dataset, config, split)
     assert "features" in response
     assert len(response["features"]) == 1
-    featureItem = response["features"][0]
-    assert "dataset" in featureItem
-    assert "config" in featureItem
-    assert "features" in featureItem
-    assert featureItem["features"]["tokens"]["_type"] == "Sequence"
+    featuresItem = response["features"][0]
+    assert "dataset" in featuresItem
+    assert "config" in featuresItem
+    assert "features" in featuresItem
+    assert featuresItem["features"]["tokens"]["_type"] == "Sequence"
 
 
 def test_get_split_rows_without_split() -> None: