Repository: huggingface/dataset-viewer

Issue:
Missing triggers between steps
possibly due to https://github.com/huggingface/datasets-server/pull/2489

We get errors like:
- https://huggingface.co/datasets/jmc255/aphantasia_drawing_dataset/discussions/1
- https://huggingface.co/datasets/yzeng58/CoBSAT/discussions/1

where the first step `dataset-config-names` gives an error, and no other entry is created, leading to the API service responding `404 - Error code:   ResponseNotFound`, while it should tell that the issue is we don't support script-based datasets anymore.




Discussion:

Resolving patch:
diff --git a/libs/libcommon/src/libcommon/processing_graph.py b/libs/libcommon/src/libcommon/processing_graph.py
--- a/libs/libcommon/src/libcommon/processing_graph.py
+++ b/libs/libcommon/src/libcommon/processing_graph.py
@@ -39,6 +39,31 @@ def guard_int(x: Any) -> int:
     raise ValueError(f"Invalid int: {x}")
 
 
+def check_one_of_parents_is_same_or_higher_level(processing_graph: ProcessingGraph) -> None:
+    first_steps = processing_graph.get_first_processing_steps()
+    for step in processing_graph.get_topologically_ordered_processing_steps():
+        parents = processing_graph.get_parents(step.name)
+        if parents:
+            parent_input_types = [p.input_type for p in parents]
+            if step.input_type == "dataset":
+                if "dataset" not in parent_input_types:
+                    raise ValueError(
+                        f"Step {step.name} is a dataset-level step but none of its parents are dataset-level."
+                    )
+            elif step.input_type == "config":
+                if "dataset" not in parent_input_types and "config" not in parent_input_types:
+                    raise ValueError(
+                        f"Step {step.name} is a config-level step but none of its parents are config-level or dataset-level."
+                    )
+            # no need to check for splits
+        else:
+            # probably redundant checks
+            if step not in first_steps:
+                raise ValueError(f"Step {step.name} has not parents, but is not a root step.")
+            if step.input_type != "dataset":
+                raise ValueError(f"Step {step.name} is a root step but is not a dataset-level step.")
+
+
 class ProcessingStepSpecification(TypedDict, total=False):
     input_type: InputType
     triggered_by: Union[list[str], str, None]
@@ -115,7 +140,11 @@ class ProcessingGraph:
     The graph can have multiple roots.
 
     Args:
-        specification (ProcessingGraphSpecification): The specification of the graph.
+        specification (`ProcessingGraphSpecification`): The specification of the graph.
+        min_bytes_for_bonus_difficulty (`int`, *optional*, defaults to `MIN_BYTES_FOR_BONUS_DIFFICULTY`): The minimum
+            number of bytes for a dataset to be considered big and get the bonus difficulty.
+        check_one_of_parents_is_same_or_higher_level (`bool`, *optional*, defaults to `True`): Whether to check that
+            one of the parents of a processing step is of the same input type or a higher input type.
 
     Raises:
         [`ValueError`]: If the graph is not a DAG.
@@ -126,6 +155,7 @@ class ProcessingGraph:
 
     specification: ProcessingGraphSpecification
     min_bytes_for_bonus_difficulty: int = MIN_BYTES_FOR_BONUS_DIFFICULTY
+    check_one_of_parents_is_same_or_higher_level: bool = True
 
     _nx_graph: nx.DiGraph = field(init=False)
     _processing_steps: Mapping[str, ProcessingStep] = field(init=False)
@@ -193,6 +223,8 @@ def __post_init__(self) -> None:
         self._alphabetically_ordered_processing_steps = [
             self.get_processing_step(processing_step_name) for processing_step_name in sorted(_nx_graph.nodes())
         ]
+        if self.check_one_of_parents_is_same_or_higher_level:
+            check_one_of_parents_is_same_or_higher_level(self)
 
     def get_processing_step(self, processing_step_name: str) -> ProcessingStep:
         """
@@ -451,6 +483,10 @@ def parse_id(id: str) -> tuple[str, str, Optional[str], Optional[str], str]:
         return dataset, revision, config, split, prefix
 
 
+# Note: apart from the root steps:
+# - every dataset-level step should be triggered by at least one dataset-level step
+# - every config-level step should be triggered by at least one dataset-level or config-level step
+# to be sure the steps are processed even if a dataset has no configs, or if a config has no splits.
 specification: ProcessingGraphSpecification = {
     "dataset-config-names": {
         "input_type": "dataset",
@@ -459,7 +495,10 @@ def parse_id(id: str) -> tuple[str, str, Optional[str], Optional[str], str]:
     },
     "split-first-rows": {
         "input_type": "split",
-        "triggered_by": ["config-split-names", "config-parquet-metadata"],
+        "triggered_by": [
+            "config-split-names",  # to compute with streaming when possible
+            "config-parquet-metadata",  # to compute with parquet
+        ],
         "job_runner_version": 4,
         "difficulty": 70,
     },
@@ -483,7 +522,10 @@ def parse_id(id: str) -> tuple[str, str, Optional[str], Optional[str], str]:
     },
     "dataset-parquet": {
         "input_type": "dataset",
-        "triggered_by": "config-parquet",
+        "triggered_by": [
+            "dataset-config-names",  # required in case the dataset has no configs (error in previous step)
+            "config-parquet",
+        ],
         "job_runner_version": 2,
         "difficulty": 20,
     },
@@ -495,13 +537,19 @@ def parse_id(id: str) -> tuple[str, str, Optional[str], Optional[str], str]:
     },
     "dataset-info": {
         "input_type": "dataset",
-        "triggered_by": "config-info",
+        "triggered_by": [
+            "dataset-config-names",  # required in case the dataset has no configs (error in previous step)
+            "config-info",
+        ],
         "job_runner_version": 2,
         "difficulty": 20,
     },
     "config-split-names": {
         "input_type": "config",
-        "triggered_by": ["dataset-config-names", "config-info"],
+        "triggered_by": [
+            "dataset-config-names",  # compute with streaming
+            "config-info",  # compute from parquet conversion
+        ],
         "job_runner_version": 3,
         "difficulty": 60,
     },
@@ -513,13 +561,19 @@ def parse_id(id: str) -> tuple[str, str, Optional[str], Optional[str], str]:
     },
     "dataset-size": {
         "input_type": "dataset",
-        "triggered_by": "config-size",
+        "triggered_by": [
+            "dataset-config-names",  # required in case the dataset has no configs (error in previous step)
+            "config-size",
+        ],
         "job_runner_version": 2,
         "difficulty": 20,
     },
     "dataset-split-names": {
         "input_type": "dataset",
-        "triggered_by": "config-split-names",
+        "triggered_by": [
+            "dataset-config-names",  # required in case the dataset has no configs (error in previous step)
+            "config-split-names",
+        ],
         "job_runner_version": 3,
         "difficulty": 20,
     },
@@ -532,19 +586,29 @@ def parse_id(id: str) -> tuple[str, str, Optional[str], Optional[str], str]:
     },
     "split-is-valid": {
         "input_type": "split",
-        "triggered_by": ["config-size", "split-first-rows", "split-duckdb-index"],
+        "triggered_by": [
+            "config-size",
+            "split-first-rows",
+            "split-duckdb-index",
+        ],
         "job_runner_version": 3,
         "difficulty": 20,
     },
     "config-is-valid": {
         "input_type": "config",
-        "triggered_by": "split-is-valid",
+        "triggered_by": [
+            "config-split-names",  # required in case the config has no splits (error in previous step)
+            "split-is-valid",
+        ],
         "job_runner_version": 2,
         "difficulty": 20,
     },
     "dataset-is-valid": {
         "input_type": "dataset",
-        "triggered_by": "config-is-valid",
+        "triggered_by": [
+            "dataset-config-names",  # required in case the dataset has no configs (error in previous step)
+            "config-is-valid",
+        ],
         "job_runner_version": 6,
         "difficulty": 20,
     },
@@ -568,13 +632,19 @@ def parse_id(id: str) -> tuple[str, str, Optional[str], Optional[str], str]:
     },
     "config-opt-in-out-urls-count": {
         "input_type": "config",
-        "triggered_by": "split-opt-in-out-urls-count",
+        "triggered_by": [
+            "config-split-names",  # required in case the config has no splits (error in previous step)
+            "split-opt-in-out-urls-count",
+        ],
         "job_runner_version": 3,
         "difficulty": 20,
     },
     "dataset-opt-in-out-urls-count": {
         "input_type": "dataset",
-        "triggered_by": "config-opt-in-out-urls-count",
+        "triggered_by": [
+            "dataset-config-names",  # required in case the dataset has no configs (error in previous step)
+            "config-opt-in-out-urls-count",
+        ],
         "job_runner_version": 2,
         "difficulty": 20,
     },
@@ -587,19 +657,29 @@ def parse_id(id: str) -> tuple[str, str, Optional[str], Optional[str], str]:
     },
     "config-duckdb-index-size": {
         "input_type": "config",
-        "triggered_by": "split-duckdb-index",
+        "triggered_by": [
+            "config-split-names",  # required in case the config has no splits (error in previous step)
+            "split-duckdb-index",
+        ],
         "job_runner_version": 2,
         "difficulty": 20,
     },
     "dataset-duckdb-index-size": {
         "input_type": "dataset",
-        "triggered_by": "config-duckdb-index-size",
+        "triggered_by": [
+            "dataset-config-names",  # required in case the dataset has no configs (error in previous step)
+            "config-duckdb-index-size",
+        ],
         "job_runner_version": 1,
         "difficulty": 20,
     },
     "dataset-hub-cache": {
         "input_type": "dataset",
-        "triggered_by": ["dataset-is-valid", "dataset-size", "dataset-loading-tags"],
+        "triggered_by": [
+            "dataset-is-valid",
+            "dataset-size",
+            "dataset-loading-tags",
+        ],
         "job_runner_version": 1,
         "difficulty": 20,
     },


Test patch:
diff --git a/libs/libcommon/tests/test_operations.py b/libs/libcommon/tests/test_operations.py
--- a/libs/libcommon/tests/test_operations.py
+++ b/libs/libcommon/tests/test_operations.py
@@ -346,7 +346,7 @@ def test_2274_only_one_entry(
         }
         finish_job(job_result=job_result)
 
-        assert len(queue.get_pending_jobs_df(dataset=dataset)) == 0
+        assert len(queue.get_pending_jobs_df(dataset=dataset)) == 7
         assert len(get_cache_entries_df(dataset=dataset)) == 1
 
         # let's delete all the jobs, to get in the same state as the bug
diff --git a/libs/libcommon/tests/test_processing_graph.py b/libs/libcommon/tests/test_processing_graph.py
--- a/libs/libcommon/tests/test_processing_graph.py
+++ b/libs/libcommon/tests/test_processing_graph.py
@@ -55,6 +55,13 @@ def test_graph() -> None:
             [
                 "config-split-names",
                 "config-parquet-and-info",
+                "dataset-duckdb-index-size",
+                "dataset-opt-in-out-urls-count",
+                "dataset-split-names",
+                "dataset-parquet",
+                "dataset-info",
+                "dataset-size",
+                "dataset-is-valid",
             ],
             [],
             [],
@@ -72,9 +79,12 @@ def test_graph() -> None:
         (
             "config-split-names",
             [
+                "config-duckdb-index-size",
+                "config-opt-in-out-urls-count",
                 "split-first-rows",
                 "dataset-split-names",
                 "split-descriptive-statistics",
+                "config-is-valid",
             ],
             ["dataset-config-names", "config-info"],
             ["dataset-config-names", "config-parquet-and-info", "config-info"],
@@ -83,6 +93,7 @@ def test_graph() -> None:
             "dataset-split-names",
             [],
             [
+                "dataset-config-names",
                 "config-split-names",
             ],
             [
@@ -120,19 +131,19 @@ def test_graph() -> None:
         (
             "dataset-parquet",
             [],
-            ["config-parquet"],
+            ["dataset-config-names", "config-parquet"],
             ["dataset-config-names", "config-parquet-and-info", "config-parquet"],
         ),
         (
             "config-info",
-            ["config-split-names", "dataset-info"],
+            ["dataset-info", "config-split-names"],
             ["config-parquet-and-info"],
             ["dataset-config-names", "config-parquet-and-info"],
         ),
         (
             "dataset-info",
             ["dataset-loading-tags"],
-            ["config-info"],
+            ["dataset-config-names", "config-info"],
             ["dataset-config-names", "config-parquet-and-info", "config-info"],
         ),
         (
@@ -144,7 +155,7 @@ def test_graph() -> None:
         (
             "dataset-size",
             ["dataset-hub-cache"],
-            ["config-size"],
+            ["dataset-config-names", "config-size"],
             ["dataset-config-names", "config-parquet-and-info", "config-size"],
         ),
         (
@@ -158,6 +169,7 @@ def test_graph() -> None:
             ["dataset-hub-cache"],
             [
                 "config-is-valid",
+                "dataset-config-names",
             ],
             [
                 "dataset-config-names",
@@ -221,7 +233,7 @@ def test_graph() -> None:
         (
             "config-opt-in-out-urls-count",
             ["dataset-opt-in-out-urls-count"],
-            ["split-opt-in-out-urls-count"],
+            ["split-opt-in-out-urls-count", "config-split-names"],
             [
                 "dataset-config-names",
                 "config-split-names",
@@ -238,7 +250,7 @@ def test_graph() -> None:
         (
             "dataset-opt-in-out-urls-count",
             [],
-            ["config-opt-in-out-urls-count"],
+            ["config-opt-in-out-urls-count", "dataset-config-names"],
             [
                 "dataset-config-names",
                 "config-split-names",
@@ -267,11 +279,13 @@ def test_graph() -> None:
         (
             "config-duckdb-index-size",
             ["dataset-duckdb-index-size"],
-            ["split-duckdb-index"],
+            ["split-duckdb-index", "config-split-names"],
             [
+                "config-split-names",
                 "config-parquet",
                 "config-parquet-and-info",
                 "config-parquet-metadata",
+                "config-info",
                 "dataset-config-names",
                 "split-duckdb-index",
             ],
@@ -279,12 +293,14 @@ def test_graph() -> None:
         (
             "dataset-duckdb-index-size",
             [],
-            ["config-duckdb-index-size"],
+            ["config-duckdb-index-size", "dataset-config-names"],
             [
                 "config-duckdb-index-size",
+                "config-split-names",
                 "config-parquet",
                 "config-parquet-and-info",
                 "config-parquet-metadata",
+                "config-info",
                 "dataset-config-names",
                 "split-duckdb-index",
             ],
diff --git a/libs/libcommon/tests/utils.py b/libs/libcommon/tests/utils.py
--- a/libs/libcommon/tests/utils.py
+++ b/libs/libcommon/tests/utils.py
@@ -187,7 +187,8 @@
         STEP_DE: {"input_type": "dataset", "triggered_by": STEP_CA},  # fan-in (C -> D)
         STEP_CB: {"input_type": "config", "triggered_by": STEP_SA},  # fan-in (S -> C)
         STEP_DF: {"input_type": "dataset", "triggered_by": STEP_SA},  # fan-in (S -> D)
-    }
+    },
+    check_one_of_parents_is_same_or_higher_level=False,
 )
 
 # Graph to test parallel steps (ie. two steps that compute the same thing, and abort if the other already exists)
diff --git a/services/worker/tests/test_job_manager.py b/services/worker/tests/test_job_manager.py
--- a/services/worker/tests/test_job_manager.py
+++ b/services/worker/tests/test_job_manager.py
@@ -142,7 +142,10 @@ def test_backfill(priority: Priority, app_config: AppConfig) -> None:
 
     child = processing_graph.get_children("dataset-config-names").pop()
     dataset_child_jobs = queue.get_dump_with_status(job_type=child.job_type, status=Status.WAITING)
-    assert len(dataset_child_jobs) == 0
+    assert len(dataset_child_jobs) == 1
+    assert dataset_child_jobs[0]["dataset"] == "dataset"
+    assert dataset_child_jobs[0]["revision"] == "revision"
+    assert dataset_child_jobs[0]["priority"] is priority.value
 
 
 def test_job_runner_set_crashed(app_config: AppConfig) -> None: