Repository: huggingface/dataset-viewer

Issue:
An error in a step should be propagated in the dependent steps
See https://github.com/huggingface/datasets-server/issues/948:
- the entries of "/first-rows" should have been deleted because the upstream jobs are errors and we cannot get the list of splits.
- the entry of "/dataset-info" should be an error, since the result of "/parquet-and-dataset-info" is an error, and both are at dataset level.
- etc.

I think we have to review the logic of what occurs in case of error, to have a coherent status for the cache of a dataset.


Discussion:
See https://github.com/huggingface/datasets-server/issues/1012
https://github.com/huggingface/datasets-server/issues/1022

Resolving patch:
diff --git a/libs/libcommon/src/libcommon/simple_cache.py b/libs/libcommon/src/libcommon/simple_cache.py
--- a/libs/libcommon/src/libcommon/simple_cache.py
+++ b/libs/libcommon/src/libcommon/simple_cache.py
@@ -329,13 +329,6 @@ def get_best_response(
     return best_response_candidates[max_index]
 
 
-def get_split_full_names_for_dataset_and_kind(dataset: str, kind: str) -> set[SplitFullName]:
-    return {
-        SplitFullName(dataset=response.dataset, config=response.config, split=response.split)
-        for response in CachedResponse.objects(dataset=dataset, kind=kind).only("dataset", "config", "split")
-    }
-
-
 def get_valid_datasets(kind: str) -> Set[str]:
     return set(CachedResponse.objects(kind=kind, http_status=HTTPStatus.OK).distinct("dataset"))
 
diff --git a/services/worker/src/worker/job_runner.py b/services/worker/src/worker/job_runner.py
--- a/services/worker/src/worker/job_runner.py
+++ b/services/worker/src/worker/job_runner.py
@@ -21,11 +21,9 @@
     CacheEntryWithDetails,
     DoesNotExist,
     SplitFullName,
-    delete_response,
     get_best_response,
     get_response,
     get_response_without_content,
-    get_split_full_names_for_dataset_and_kind,
     upsert_response,
 )
 from libcommon.utils import orjson_dumps
@@ -544,16 +542,25 @@ def create_children_jobs(self) -> None:
         except Exception:
             # if the response is not in the cache, we don't create the children jobs
             return
-        if response_in_cache["http_status"] != HTTPStatus.OK:
-            # if the response is not valid, we don't create the children jobs
-            return
-        new_split_full_names_for_split: set[SplitFullName] = self.get_new_splits(response_in_cache["content"])
-        new_split_full_names_for_config: set[SplitFullName] = {
-            SplitFullName(dataset=s.dataset, config=s.config, split=None) for s in new_split_full_names_for_split
-        }
-        new_split_full_names_for_dataset: set[SplitFullName] = {
-            SplitFullName(dataset=s.dataset, config=None, split=None) for s in new_split_full_names_for_config
-        }  # should be self.dataset
+        if response_in_cache["http_status"] == HTTPStatus.OK:
+            new_split_full_names_for_split: set[SplitFullName] = self.get_new_splits(response_in_cache["content"])
+            new_split_full_names_for_config: set[SplitFullName] = {
+                SplitFullName(dataset=s.dataset, config=s.config, split=None) for s in new_split_full_names_for_split
+            }
+        elif self.processing_step.input_type == "split":
+            new_split_full_names_for_split = {
+                SplitFullName(dataset=self.dataset, config=self.config, split=self.split)
+            }
+            new_split_full_names_for_config = {SplitFullName(dataset=self.dataset, config=self.config, split=None)}
+        elif self.processing_step.input_type == "config":
+            new_split_full_names_for_split = set()
+            new_split_full_names_for_config = {SplitFullName(dataset=self.dataset, config=self.config, split=None)}
+
+        else:
+            new_split_full_names_for_split = set()
+            new_split_full_names_for_config = set()
+        new_split_full_names_for_dataset = {SplitFullName(dataset=self.dataset, config=None, split=None)}
+
         for processing_step in self.processing_step.children:
             new_split_full_names = (
                 new_split_full_names_for_split
@@ -562,22 +569,6 @@ def create_children_jobs(self) -> None:
                 if processing_step.input_type == "config"
                 else new_split_full_names_for_dataset
             )
-            # remove obsolete responses from the cache
-            split_full_names_in_cache = get_split_full_names_for_dataset_and_kind(
-                dataset=self.dataset, kind=processing_step.cache_kind
-            )
-            split_full_names_to_delete = split_full_names_in_cache.difference(new_split_full_names)
-            for split_full_name in split_full_names_to_delete:
-                delete_response(
-                    kind=processing_step.cache_kind,
-                    dataset=split_full_name.dataset,
-                    config=split_full_name.config,
-                    split=split_full_name.split,
-                )
-            logging.debug(
-                f"{len(split_full_names_to_delete)} obsolete responses"
-                f"of kind {processing_step.cache_kind} deleted from cache for dataset={self.dataset}"
-            )
             # compute the responses for the new splits
             queue = Queue()
             for split_full_name in new_split_full_names:


Test patch:
diff --git a/libs/libcommon/tests/test_simple_cache.py b/libs/libcommon/tests/test_simple_cache.py
--- a/libs/libcommon/tests/test_simple_cache.py
+++ b/libs/libcommon/tests/test_simple_cache.py
@@ -17,7 +17,6 @@
     DoesNotExist,
     InvalidCursor,
     InvalidLimit,
-    SplitFullName,
     delete_dataset_responses,
     delete_response,
     get_best_response,
@@ -28,7 +27,6 @@
     get_response,
     get_response_without_content,
     get_responses_count_by_kind_status_and_error_code,
-    get_split_full_names_for_dataset_and_kind,
     get_valid_datasets,
     get_validity_by_kind,
     upsert_response,
@@ -214,45 +212,6 @@ def test_big_row() -> None:
         )
 
 
-def test_get_split_full_names_for_dataset_and_kind() -> None:
-    kind_a = "test_kind_a"
-    kind_b = "test_kind_b"
-    dataset_a = "test_dataset_a"
-    dataset_b = "test_dataset_b"
-    dataset_c = "test_dataset_c"
-    config_a = "test_config_a"
-    config_b = "test_config_b"
-    split_a = "test_split_a"
-    split_b = None
-    upsert_response(kind=kind_a, dataset=dataset_a, content={}, http_status=HTTPStatus.OK)
-    upsert_response(
-        kind=kind_b, dataset=dataset_a, config=config_a, split=split_a, content={}, http_status=HTTPStatus.OK
-    )
-    upsert_response(
-        kind=kind_b, dataset=dataset_a, config=config_b, split=split_a, content={}, http_status=HTTPStatus.OK
-    )
-    upsert_response(
-        kind=kind_b, dataset=dataset_a, config=config_b, split=split_b, content={}, http_status=HTTPStatus.OK
-    )
-    upsert_response(kind=kind_a, dataset=dataset_b, content={}, http_status=HTTPStatus.OK)
-    result = get_split_full_names_for_dataset_and_kind(dataset=dataset_a, kind=kind_a)
-    expected = {SplitFullName(dataset_a, None, None)}
-    assert len(result) == len(expected) and all(x in expected for x in result)
-    # ^ compare the contents of the lists without caring about the order
-    result = get_split_full_names_for_dataset_and_kind(dataset=dataset_a, kind=kind_b)
-    expected = {
-        SplitFullName(dataset_a, config_a, split_a),
-        SplitFullName(dataset_a, config_b, split_b),
-        SplitFullName(dataset_a, config_b, split_a),
-    }
-    assert len(result) == len(expected) and all(x in expected for x in result)
-    # ^ compare the contents of the lists without caring about the order
-    assert get_split_full_names_for_dataset_and_kind(dataset=dataset_b, kind=kind_a) == {
-        SplitFullName(dataset_b, None, None)
-    }
-    assert get_split_full_names_for_dataset_and_kind(dataset=dataset_c, kind=kind_a) == set()
-
-
 def test_get_valid_dataset_names_empty() -> None:
     assert not get_valid_datasets(kind="test_kind")