Repository: huggingface/dataset-viewer

Issue:
Cron job that cleans hf-datasets cache often fails
> Description: Job datasets-server/prod-datasets-server-job-clean-hf-datasets-cache-28426140 failed to complete. Removing failed job after investigation should clear this alert.

https://huggingface.slack.com/archives/C05UE1R5ZBJ/p1705571880300879




Discussion:
Logs:

```
ERROR: 2024-01-18 09:22:30,433 - root - [Errno 2] No such file or directory: '/storage/hf-datasets-cache/medium/datasets/17158051218505-dataset-config-names-jacobbieker-eumetsat-0deg-30e327e0'
```

```
FileNotFoundError: [Errno 2] No such file or directory: '/storage/hf-datasets-cache/medium/datasets/17158051218505-dataset-config-names-jacobbieker-eumetsat-0deg-30e327e0'
```
The issue comes from here:

https://github.com/huggingface/datasets-server/blob/25b2293a7bf478d76b36e99671205b74165d82e0/jobs/cache_maintenance/src/cache_maintenance/clean_directory.py#L29

Resolving patch:
diff --git a/jobs/cache_maintenance/src/cache_maintenance/clean_directory.py b/jobs/cache_maintenance/src/cache_maintenance/clean_directory.py
--- a/jobs/cache_maintenance/src/cache_maintenance/clean_directory.py
+++ b/jobs/cache_maintenance/src/cache_maintenance/clean_directory.py
@@ -21,26 +21,35 @@ def clean_directory(pattern: str, expired_time_interval_seconds: int) -> None:
     errors = 0
     total_dirs = 0
     total_files = 0
+    total_disappeared_paths = 0
 
     def rmtree_on_error(function: Any, path: str, excinfo: Any) -> None:  # noqa: ARG001, args needed for onerror=
         logging.error(f"failed to delete directory {path=}")
 
     for path in glob.glob(pattern):
-        last_access_time_value = os.path.getatime(path)
-        last_access_datetime = datetime.fromtimestamp(last_access_time_value).replace(tzinfo=None)
-        if last_access_datetime + timedelta(seconds=expired_time_interval_seconds) <= now:
-            if os.path.isfile(path):
-                logging.info(f"deleting file {path=} {last_access_datetime=}")
-                os.remove(path)
-                total_files += 1
-            elif os.path.isdir(path):
-                logging.info(f"deleting directory {path=} {last_access_datetime=}")
-                shutil.rmtree(path, onerror=rmtree_on_error)
-                total_dirs += 1
-                errors += os.path.isdir(path)
+        try:
+            last_access_time_value = os.path.getatime(path)
+            last_access_datetime = datetime.fromtimestamp(last_access_time_value).replace(tzinfo=None)
+            if last_access_datetime + timedelta(seconds=expired_time_interval_seconds) <= now:
+                if os.path.isfile(path):
+                    logging.info(f"deleting file {path=} {last_access_datetime=}")
+                    os.remove(path)
+                    total_files += 1
+                elif os.path.isdir(path):
+                    logging.info(f"deleting directory {path=} {last_access_datetime=}")
+                    shutil.rmtree(path, onerror=rmtree_on_error)
+                    total_dirs += 1
+                    errors += os.path.isdir(path)
+        except FileNotFoundError:
+            logging.error(f"failed to delete {path=} because it has disappeared during the loop")
+            total_disappeared_paths += 1
     if errors:
         logging.error(f"clean_directory failed to remove {errors} directories at the root of the cache directory.")
     if total_files:
         logging.info(f"clean_directory removed {total_files} files at the root of the cache directory.")
+    if total_disappeared_paths:
+        logging.info(
+            f"clean_directory failed to delete {total_disappeared_paths} paths because they disappeared during the loop."
+        )
 
     logging.info(f"clean_directory removed {total_dirs - errors} directories at the root of the cache directory.")


Test patch: