Repository: huggingface/dataset-viewer

Issue:
Limit the number of fields (columns) in /croissant
https://huggingface.co/datasets/mstz/arcene has 10K columns, and thus the /croissant response is 3.7MB (120K lines when the JSON is prettified).

We already limit the number of supported columns to 1K: https://github.com/huggingface/datasets-server/blob/main/chart/values.yaml#L204

We could apply the same limit, and mention it in the description of the recordset.

From Slack (internal): https://huggingface.slack.com/archives/C04L6P8KNQ5/p1705224717632239 


Discussion:

Resolving patch:
diff --git a/services/api/src/api/routes/croissant.py b/services/api/src/api/routes/croissant.py
--- a/services/api/src/api/routes/croissant.py
+++ b/services/api/src/api/routes/croissant.py
@@ -28,6 +28,9 @@
 from starlette.responses import Response
 
 MAX_CONFIGS = 100
+MAX_COLUMNS = 1_000
+# ^ same value as the default for FIRST_ROWS_COLUMNS_MAX_NUMBER (see services/worker)
+
 
 HF_TO_CROISSANT_VALUE_TYPE = {
     "string": "sc:Text",
@@ -76,6 +79,7 @@ def get_croissant_from_dataset_infos(dataset: str, infos: list[Mapping[str, Any]
     ]
     record_set = []
     for info in infos:
+        description_body = ""
         config = info["config_name"]
         features = Features.from_dict(info["features"])
         fields: list[dict[str, Any]] = []
@@ -92,6 +96,9 @@ def get_croissant_from_dataset_infos(dataset: str, infos: list[Mapping[str, Any]
         )
         skipped_columns = []
         for column, feature in features.items():
+            if len(fields) >= MAX_COLUMNS:
+                description_body += f"\n- {len(features) - MAX_COLUMNS} skipped column{'s' if len(features) - MAX_COLUMNS > 1 else ''} (max number of columns reached)"
+                break
             fields_names: set[str] = set()
             if isinstance(feature, Value) and feature.dtype in HF_TO_CROISSANT_VALUE_TYPE:
                 fields.append(
@@ -134,7 +141,6 @@ def get_croissant_from_dataset_infos(dataset: str, infos: list[Mapping[str, Any]
         description = f"{dataset} - '{config}' subset"
         if partial:
             description += " (first 5GB)"
-        description_body = ""
         if len(splits) > 1:
             description_body += f"\n- {len(splits)} split{'s' if len(splits) > 1 else ''}: {', '.join(splits)}"
         if skipped_columns:


Test patch:
diff --git a/services/api/tests/routes/test_croissant.py b/services/api/tests/routes/test_croissant.py
--- a/services/api/tests/routes/test_croissant.py
+++ b/services/api/tests/routes/test_croissant.py
@@ -1,3 +1,5 @@
+from unittest.mock import patch
+
 from api.routes.croissant import get_croissant_from_dataset_infos
 
 squad_info = {
@@ -63,3 +65,12 @@ def test_get_croissant_from_dataset_infos() -> None:
     assert isinstance(croissant["recordSet"][0]["field"], list)
     assert isinstance(squad_info["features"], dict)
     assert len(croissant["recordSet"][0]["field"]) == len(squad_info["features"]) - 1
+
+
+def test_get_croissant_from_dataset_infos_max_columns() -> None:
+    MAX_COLUMNS = 3
+    with patch("api.routes.croissant.MAX_COLUMNS", MAX_COLUMNS):
+        croissant = get_croissant_from_dataset_infos("user/squad with space", [squad_info, squad_info], partial=False)
+
+    assert len(croissant["recordSet"][0]["field"]) == MAX_COLUMNS
+    assert "max number of columns reached" in croissant["recordSet"][0]["description"]