Repository: huggingface/dataset-viewer

Issue:
Backfill ignores the existing cache entries for previous config/split names
See https://github.com/huggingface/dataset-viewer/issues/2766#issuecomment-2090443731

> For https://huggingface.co/datasets/re-align/UnifiedChat, for example, we have entries, including an error with LockedDatasetTimeoutError, for config `default`. But this config does not exist anymore.

In this case, the backfill process should have deleted all the cache entries for config `default`.




Discussion:
Fixing this should remove the unused cache entries. I don't think it's possible to have an idea of their number before fixing the bug.

Resolving patch:
diff --git a/libs/libcommon/src/libcommon/operations.py b/libs/libcommon/src/libcommon/operations.py
--- a/libs/libcommon/src/libcommon/operations.py
+++ b/libs/libcommon/src/libcommon/operations.py
@@ -23,6 +23,7 @@
     NotSupportedRepositoryNotFoundError,
 )
 from libcommon.orchestrator import TasksStatistics, backfill, get_revision, remove_dataset, set_revision
+from libcommon.state import IncoherentCacheError
 from libcommon.storage_client import StorageClient
 from libcommon.utils import raise_if_blocked
 
@@ -260,11 +261,14 @@ def backfill_dataset(
     except NotSupportedError as e:
         logging.warning(f"Dataset {dataset} is not supported ({type(e)}). Let's delete the dataset.")
         return delete_dataset(dataset=dataset, storage_clients=storage_clients)
-    tasks_statistics = backfill(
-        dataset=dataset,
-        revision=revision,
-        priority=priority,
-    )
+    try:
+        tasks_statistics = backfill(dataset=dataset, revision=revision, priority=priority)
+    except IncoherentCacheError:
+        logging.warning(
+            f"Dataset {dataset} has incoherent entries in the cache. Let's first delete the dataset, then backfill again."
+        )
+        delete_dataset(dataset=dataset, storage_clients=storage_clients)
+        tasks_statistics = backfill(dataset=dataset, revision=revision, priority=priority)
     has_tasks = tasks_statistics.has_tasks()
     return OperationsStatistics(
         num_backfilled_datasets=1 if has_tasks else 0,
diff --git a/libs/libcommon/src/libcommon/state.py b/libs/libcommon/src/libcommon/state.py
--- a/libs/libcommon/src/libcommon/state.py
+++ b/libs/libcommon/src/libcommon/state.py
@@ -3,6 +3,7 @@
 
 import logging
 from dataclasses import dataclass, field
+from itertools import islice
 from typing import Optional
 
 import pandas as pd
@@ -19,6 +20,18 @@
 # TODO: assets, cached_assets, parquet files
 
 
+class IncoherentCacheError(Exception):
+    pass
+
+
+class UnexceptedConfigNamesError(IncoherentCacheError):
+    pass
+
+
+class UnexceptedSplitNamesError(IncoherentCacheError):
+    pass
+
+
 @dataclass
 class JobState:
     """The state of a job for a given input."""
@@ -219,6 +232,14 @@ def __post_init__(self) -> None:
                 name_field="split",
             )  # Note that we use the cached content even the revision is different (ie. maybe obsolete)
 
+        unexpected_split_names = set(self.cache_entries_df["split"].unique()).difference(
+            set(self.split_names).union({None})
+        )
+        if unexpected_split_names:
+            raise UnexceptedSplitNamesError(
+                f"Unexpected split names for dataset={self.dataset} config={self.config} ({len(unexpected_split_names)}): {list(islice(unexpected_split_names, 10))}{'' if len(unexpected_split_names) <= 10 else '...'}"
+            )
+
         with StepProfiler(
             method="ConfigState.__post_init__",
             step="get_split_states",
@@ -226,10 +247,10 @@ def __post_init__(self) -> None:
         ):
             self.split_states = [
                 SplitState(
-                    self.dataset,
-                    self.revision,
-                    self.config,
-                    split_name,
+                    dataset=self.dataset,
+                    revision=self.revision,
+                    config=self.config,
+                    split=split_name,
                     processing_graph=self.processing_graph,
                     pending_jobs_df=self.pending_jobs_df[self.pending_jobs_df["split"] == split_name],
                     cache_entries_df=self.cache_entries_df[self.cache_entries_df["split"] == split_name],
@@ -293,6 +314,14 @@ def __post_init__(self) -> None:
                     name_field="config",
                 )  # Note that we use the cached content even the revision is different (ie. maybe obsolete)
 
+            unexpected_config_names = set(self.cache_entries_df["config"].unique()).difference(
+                set(self.config_names).union({None})
+            )
+            if unexpected_config_names:
+                raise UnexceptedConfigNamesError(
+                    f"Unexpected config names ({len(unexpected_config_names)}): {list(islice(unexpected_config_names, 10))}{'' if len(unexpected_config_names) <= 10 else '...'}"
+                )
+
             with StepProfiler(
                 method="DatasetState.__post_init__",
                 step="get_config_states",


Test patch:
diff --git a/libs/libcommon/tests/test_backfill.py b/libs/libcommon/tests/test_backfill.py
--- a/libs/libcommon/tests/test_backfill.py
+++ b/libs/libcommon/tests/test_backfill.py
@@ -12,6 +12,7 @@
 from libcommon.processing_graph import ProcessingGraph
 from libcommon.queue import Queue
 from libcommon.resources import CacheMongoResource, QueueMongoResource
+from libcommon.state import UnexceptedConfigNamesError
 
 from .utils import (
     ARTIFACT_CA_1,
@@ -402,10 +403,13 @@ def test_plan_retry_error_and_outdated_by_parent(
     compute_all(processing_graph=processing_graph)
 
     put_cache(step=STEP_DA, dataset=DATASET_NAME, revision=REVISION_NAME, error_code=error_code)
-    # in the case of PROCESSING_GRAPH_FAN_IN_OUT: the config names do not exist anymore:
-    # the cache entries (also the jobs, if any - not here) should be deleted.
-    # they are still here, and haunting the database
-    # TODO: Not supported yet
+    if processing_graph == PROCESSING_GRAPH_FAN_IN_OUT:
+        # in the case of PROCESSING_GRAPH_FAN_IN_OUT: the config names do not exist anymore:
+        # the cache entries (also the jobs, if any - not here) should be deleted.
+        # UnexceptedConfigNamesError is raised
+        with pytest.raises(UnexceptedConfigNamesError):
+            get_dataset_backfill_plan(processing_graph=processing_graph)
+        return
 
     dataset_backfill_plan = get_dataset_backfill_plan(processing_graph=processing_graph)
     assert_dataset_backfill_plan(
@@ -677,6 +681,14 @@ def test_plan_incoherent_state(
         else:
             raise NotImplementedError()
 
+    if processing_graph == PROCESSING_GRAPH_FAN_IN_OUT:
+        # in the case of PROCESSING_GRAPH_FAN_IN_OUT: the config names do not exist anymore:
+        # the cache entries (also the jobs, if any - not here) should be deleted.
+        # UnexceptedConfigNamesError is raised
+        with pytest.raises(UnexceptedConfigNamesError):
+            get_dataset_backfill_plan(processing_graph=processing_graph)
+        return
+
     dataset_backfill_plan = get_dataset_backfill_plan(processing_graph=processing_graph)
     assert_dataset_backfill_plan(
         dataset_backfill_plan=dataset_backfill_plan,
diff --git a/libs/libcommon/tests/test_backfill_on_real_graph.py b/libs/libcommon/tests/test_backfill_on_real_graph.py
--- a/libs/libcommon/tests/test_backfill_on_real_graph.py
+++ b/libs/libcommon/tests/test_backfill_on_real_graph.py
@@ -5,14 +5,17 @@
 
 import pytest
 
-from libcommon.processing_graph import processing_graph
+from libcommon.constants import CONFIG_SPLIT_NAMES_KIND, DATASET_CONFIG_NAMES_KIND
+from libcommon.processing_graph import processing_graph, specification
 from libcommon.queue import Queue
 from libcommon.resources import CacheMongoResource, QueueMongoResource
 from libcommon.simple_cache import upsert_response
+from libcommon.state import UnexceptedConfigNamesError, UnexceptedSplitNamesError
 
 from .utils import (
     CONFIG_NAMES,
     CONFIG_NAMES_CONTENT,
+    DATASET_NAME,
     REVISION_NAME,
     assert_dataset_backfill_plan,
     get_dataset_backfill_plan,
@@ -200,3 +203,70 @@ def test_plan_job_creation_and_termination() -> None:
         },
         tasks=["CreateJobs,18"],
     )
+
+
+def test_backfill_should_delete_inexistent_configs() -> None:
+    # see https://github.com/huggingface/dataset-viewer/issues/2767
+
+    # the list of configs is set to ["config1", "config2"]
+    upsert_response(
+        kind=DATASET_CONFIG_NAMES_KIND,
+        dataset=DATASET_NAME,
+        content=CONFIG_NAMES_CONTENT,
+        http_status=HTTPStatus.OK,
+        job_runner_version=specification[DATASET_CONFIG_NAMES_KIND]["job_runner_version"],
+        dataset_git_revision=REVISION_NAME,
+    )
+    # let's add a child artifact for an inexistent config name
+    upsert_response(
+        kind=CONFIG_SPLIT_NAMES_KIND,
+        dataset=DATASET_NAME,
+        config="does-not-exist",
+        content={},
+        http_status=HTTPStatus.OK,
+        job_runner_version=specification[CONFIG_SPLIT_NAMES_KIND]["job_runner_version"],
+        dataset_git_revision=REVISION_NAME,
+    )
+
+    # creating the backfill plan should raise UnexceptedConfigNamesError (which means we should recreate the dataset)
+    with pytest.raises(UnexceptedConfigNamesError):
+        get_dataset_backfill_plan(processing_graph=processing_graph)
+
+
+def test_backfill_should_delete_inexistent_splits() -> None:
+    # see https://github.com/huggingface/dataset-viewer/issues/2767
+
+    config = "config"
+    upsert_response(
+        kind=DATASET_CONFIG_NAMES_KIND,
+        dataset=DATASET_NAME,
+        content={"config_names": [{"dataset": DATASET_NAME, "config": config}]},
+        http_status=HTTPStatus.OK,
+        job_runner_version=specification[DATASET_CONFIG_NAMES_KIND]["job_runner_version"],
+        dataset_git_revision=REVISION_NAME,
+    )
+    # the list of splits is set to ["split_ok"]
+    upsert_response(
+        kind=CONFIG_SPLIT_NAMES_KIND,
+        dataset=DATASET_NAME,
+        config=config,
+        content={"splits": [{"dataset": DATASET_NAME, "config": config, "split": "split_ok"}]},
+        http_status=HTTPStatus.OK,
+        job_runner_version=specification[CONFIG_SPLIT_NAMES_KIND]["job_runner_version"],
+        dataset_git_revision=REVISION_NAME,
+    )
+    # let's add a child artifact for an inexistent config name
+    upsert_response(
+        kind="split-first-rows",
+        dataset=DATASET_NAME,
+        config=config,
+        split="does-not-exist",
+        content={},
+        http_status=HTTPStatus.OK,
+        job_runner_version=specification["split-first-rows"]["job_runner_version"],
+        dataset_git_revision=REVISION_NAME,
+    )
+
+    # creating the backfill plan should raise UnexceptedSplitNamesError (which means we should recreate the dataset)
+    with pytest.raises(UnexceptedSplitNamesError):
+        get_dataset_backfill_plan(processing_graph=processing_graph)
diff --git a/libs/libcommon/tests/test_operations.py b/libs/libcommon/tests/test_operations.py
--- a/libs/libcommon/tests/test_operations.py
+++ b/libs/libcommon/tests/test_operations.py
@@ -14,6 +14,7 @@
 from huggingface_hub.utils import HfHubHTTPError
 from requests import Response  # type: ignore
 
+from libcommon.constants import CONFIG_SPLIT_NAMES_KIND, DATASET_CONFIG_NAMES_KIND
 from libcommon.dtos import JobResult
 from libcommon.exceptions import (
     DatasetInBlockListError,
@@ -29,6 +30,7 @@
     update_dataset,
 )
 from libcommon.orchestrator import finish_job
+from libcommon.processing_graph import specification
 from libcommon.queue import Queue
 from libcommon.resources import CacheMongoResource, QueueMongoResource
 from libcommon.simple_cache import get_cache_entries_df, has_some_cache, upsert_response
@@ -367,3 +369,108 @@ def test_2274_only_one_entry(
 
         assert not queue.get_pending_jobs_df(dataset=dataset).empty
         assert len(get_cache_entries_df(dataset=dataset)) == 1
+
+
+@pytest.mark.parametrize(
+    "config_names,should_be_recreated",
+    [(["config_ok"], False), (["config_nok"], True), (["config_ok", "config_nok"], True)],
+)
+def test_2767_delete_inexistent_configs(
+    queue_mongo_resource: QueueMongoResource,
+    cache_mongo_resource: CacheMongoResource,
+    config_names: list[str],
+    should_be_recreated: bool,
+) -> None:
+    # see https://github.com/huggingface/dataset-viewer/issues/2767
+    with tmp_dataset(namespace=NORMAL_USER, token=NORMAL_USER_TOKEN, private=False) as dataset:
+        revision = get_latest_dataset_revision_if_supported_or_raise(
+            dataset=dataset,
+            hf_endpoint=CI_HUB_ENDPOINT,
+            hf_token=NORMAL_USER_TOKEN,
+        )
+        upsert_response(
+            kind=DATASET_CONFIG_NAMES_KIND,
+            dataset=dataset,
+            content={"config_names": [{"dataset": dataset, "config": "config_ok"}]},
+            http_status=HTTPStatus.OK,
+            job_runner_version=specification[DATASET_CONFIG_NAMES_KIND]["job_runner_version"],
+            dataset_git_revision=revision,
+        )
+        for config_name in config_names:
+            upsert_response(
+                kind=CONFIG_SPLIT_NAMES_KIND,
+                dataset=dataset,
+                config=config_name,
+                content={"not": "important"},
+                http_status=HTTPStatus.OK,
+                job_runner_version=specification[CONFIG_SPLIT_NAMES_KIND]["job_runner_version"],
+                dataset_git_revision=revision,
+            )
+        operation_statistics = backfill_dataset(dataset=dataset, hf_endpoint=CI_HUB_ENDPOINT, hf_token=CI_APP_TOKEN)
+        assert operation_statistics.num_backfilled_datasets == 1
+        # ^ when the config names contain something else than "config_ok", the dataset should have been deleted in the cache,
+        # and the first job should have been created again
+        if should_be_recreated:
+            assert Queue().has_pending_jobs(dataset=dataset, job_types=[DATASET_CONFIG_NAMES_KIND])
+            assert get_cache_entries_df(dataset=dataset).empty
+        else:
+            assert not Queue().has_pending_jobs(dataset=dataset, job_types=[DATASET_CONFIG_NAMES_KIND])
+            assert len(get_cache_entries_df(dataset=dataset)) == len(config_names) + 1
+
+
+@pytest.mark.parametrize(
+    "split_names,should_be_recreated",
+    [(["split_ok"], False), (["split_nok"], True), (["split_ok", "split_nok"], True)],
+)
+def test_2767_delete_inexistent_splits(
+    queue_mongo_resource: QueueMongoResource,
+    cache_mongo_resource: CacheMongoResource,
+    split_names: list[str],
+    should_be_recreated: bool,
+) -> None:
+    # see https://github.com/huggingface/dataset-viewer/issues/2767
+    with tmp_dataset(namespace=NORMAL_USER, token=NORMAL_USER_TOKEN, private=False) as dataset:
+        config = "config"
+        revision = get_latest_dataset_revision_if_supported_or_raise(
+            dataset=dataset,
+            hf_endpoint=CI_HUB_ENDPOINT,
+            hf_token=NORMAL_USER_TOKEN,
+        )
+        upsert_response(
+            kind=DATASET_CONFIG_NAMES_KIND,
+            dataset=dataset,
+            content={"config_names": [{"dataset": dataset, "config": config}]},
+            http_status=HTTPStatus.OK,
+            job_runner_version=specification[DATASET_CONFIG_NAMES_KIND]["job_runner_version"],
+            dataset_git_revision=revision,
+        )
+        upsert_response(
+            kind=CONFIG_SPLIT_NAMES_KIND,
+            dataset=dataset,
+            config=config,
+            content={"splits": [{"dataset": dataset, "config": config, "split": "split_ok"}]},
+            http_status=HTTPStatus.OK,
+            job_runner_version=specification[CONFIG_SPLIT_NAMES_KIND]["job_runner_version"],
+            dataset_git_revision=revision,
+        )
+        for split_name in split_names:
+            upsert_response(
+                kind="split-first-rows",
+                dataset=dataset,
+                config=config,
+                split=split_name,
+                content={"not": "important"},
+                http_status=HTTPStatus.OK,
+                job_runner_version=specification["split-first-rows"]["job_runner_version"],
+                dataset_git_revision=revision,
+            )
+        operation_statistics = backfill_dataset(dataset=dataset, hf_endpoint=CI_HUB_ENDPOINT, hf_token=CI_APP_TOKEN)
+        assert operation_statistics.num_backfilled_datasets == 1
+        # ^ when the split names contain something else than "split_ok", the dataset should have been deleted in the cache,
+        # and the first job should have been created again
+        if should_be_recreated:
+            assert Queue().has_pending_jobs(dataset=dataset, job_types=[DATASET_CONFIG_NAMES_KIND])
+            assert get_cache_entries_df(dataset=dataset).empty
+        else:
+            assert not Queue().has_pending_jobs(dataset=dataset, job_types=[DATASET_CONFIG_NAMES_KIND])
+            assert len(get_cache_entries_df(dataset=dataset)) == len(split_names) + 2