Repository: huggingface/dataset-viewer

Issue:
Use randomness in zombie+long jobs killing process?
Every worker runs the function to kill zombies every 10 minutes. But as all the workers are launch about at the same, it's pretty strange: nothing happens during 10 minutes, then 80 workers try to kill the zombies.

Same for the long jobs.

The interval between run could be random between 5 and 15 minutes, for example.


Discussion:

Resolving patch:
diff --git a/libs/libcommon/src/libcommon/queue.py b/libs/libcommon/src/libcommon/queue.py
--- a/libs/libcommon/src/libcommon/queue.py
+++ b/libs/libcommon/src/libcommon/queue.py
@@ -783,7 +783,7 @@ def heartbeat(self, job_id: str) -> None:
             return
         job.update(last_heartbeat=get_datetime())
 
-    def get_zombies(self, max_seconds_without_heartbeat: int) -> List[JobInfo]:
+    def get_zombies(self, max_seconds_without_heartbeat: float) -> List[JobInfo]:
         """Get the zombie jobs.
         It returns jobs without recent heartbeats, which means they crashed at one point and became zombies.
         Usually `max_seconds_without_heartbeat` is a factor of the time between two heartbeats.
diff --git a/services/worker/src/worker/config.py b/services/worker/src/worker/config.py
--- a/services/worker/src/worker/config.py
+++ b/services/worker/src/worker/config.py
@@ -36,17 +36,17 @@ def get_empty_str_list() -> List[str]:
 @dataclass(frozen=True)
 class WorkerConfig:
     content_max_bytes: int = WORKER_CONTENT_MAX_BYTES
-    heartbeat_interval_seconds: int = WORKER_HEARTBEAT_INTERVAL_SECONDS
+    heartbeat_interval_seconds: float = WORKER_HEARTBEAT_INTERVAL_SECONDS
     job_types_blocked: list[str] = field(default_factory=get_empty_str_list)
     job_types_only: list[str] = field(default_factory=get_empty_str_list)
-    kill_long_job_interval_seconds: int = WORKER_KILL_LONG_JOB_INTERVAL_SECONDS
-    kill_zombies_interval_seconds: int = WORKER_KILL_ZOMBIES_INTERVAL_SECONDS
+    kill_long_job_interval_seconds: float = WORKER_KILL_LONG_JOB_INTERVAL_SECONDS
+    kill_zombies_interval_seconds: float = WORKER_KILL_ZOMBIES_INTERVAL_SECONDS
     max_disk_usage_pct: int = WORKER_MAX_DISK_USAGE_PCT
-    max_job_duration_seconds: int = WORKER_MAX_JOB_DURATION_SECONDS
+    max_job_duration_seconds: float = WORKER_MAX_JOB_DURATION_SECONDS
     max_load_pct: int = WORKER_MAX_LOAD_PCT
     max_memory_pct: int = WORKER_MAX_MEMORY_PCT
     max_missing_heartbeats: int = WORKER_MAX_MISSING_HEARTBEATS
-    sleep_seconds: int = WORKER_SLEEP_SECONDS
+    sleep_seconds: float = WORKER_SLEEP_SECONDS
     state_file_path: Optional[str] = WORKER_STATE_FILE_PATH
     storage_paths: List[str] = field(default_factory=get_empty_str_list)
 
@@ -56,25 +56,25 @@ def from_env(cls) -> "WorkerConfig":
         with env.prefixed("WORKER_"):
             return cls(
                 content_max_bytes=env.int(name="CONTENT_MAX_BYTES", default=WORKER_CONTENT_MAX_BYTES),
-                heartbeat_interval_seconds=env.int(
+                heartbeat_interval_seconds=env.float(
                     name="HEARTBEAT_INTERVAL_SECONDS", default=WORKER_HEARTBEAT_INTERVAL_SECONDS
                 ),
                 job_types_blocked=env.list(name="JOB_TYPES_BLOCKED", default=get_empty_str_list()),
                 job_types_only=env.list(name="JOB_TYPES_ONLY", default=get_empty_str_list()),
-                kill_long_job_interval_seconds=env.int(
+                kill_long_job_interval_seconds=env.float(
                     name="KILL_LONG_JOB_INTERVAL_SECONDS", default=WORKER_KILL_LONG_JOB_INTERVAL_SECONDS
                 ),
-                kill_zombies_interval_seconds=env.int(
+                kill_zombies_interval_seconds=env.float(
                     name="KILL_ZOMBIES_INTERVAL_SECONDS", default=WORKER_KILL_ZOMBIES_INTERVAL_SECONDS
                 ),
                 max_disk_usage_pct=env.int(name="MAX_DISK_USAGE_PCT", default=WORKER_MAX_DISK_USAGE_PCT),
-                max_job_duration_seconds=env.int(
+                max_job_duration_seconds=env.float(
                     name="MAX_JOB_DURATION_SECONDS", default=WORKER_MAX_JOB_DURATION_SECONDS
                 ),
                 max_load_pct=env.int(name="MAX_LOAD_PCT", default=WORKER_MAX_LOAD_PCT),
                 max_memory_pct=env.int(name="MAX_MEMORY_PCT", default=WORKER_MAX_MEMORY_PCT),
                 max_missing_heartbeats=env.int(name="MAX_MISSING_HEARTBEATS", default=WORKER_MAX_MISSING_HEARTBEATS),
-                sleep_seconds=env.int(name="SLEEP_SECONDS", default=WORKER_SLEEP_SECONDS),
+                sleep_seconds=env.float(name="SLEEP_SECONDS", default=WORKER_SLEEP_SECONDS),
                 state_file_path=env.str(
                     name="STATE_FILE_PATH", default=WORKER_STATE_FILE_PATH
                 ),  # this environment variable is not expected to be set explicitly, it's set by the worker executor
diff --git a/services/worker/src/worker/executor.py b/services/worker/src/worker/executor.py
--- a/services/worker/src/worker/executor.py
+++ b/services/worker/src/worker/executor.py
@@ -5,7 +5,8 @@
 import os
 import sys
 from datetime import datetime, timedelta
-from typing import Any, Callable, Optional
+from random import random
+from typing import Any, Callable, Optional, Tuple, Union
 
 import orjson
 from filelock import FileLock
@@ -24,13 +25,20 @@
 
 
 async def every(
-    func: Callable[..., Optional[Any]], *args: Any, seconds: int, stop_on: Optional[Any] = None, **kwargs: Any
+    func: Callable[..., Optional[Any]],
+    *args: Any,
+    seconds: Union[float, Tuple[float, float]],
+    stop_on: Optional[Any] = None,
+    **kwargs: Any,
 ) -> None:
     while True:
         out = func(*args, **kwargs)
         if stop_on is not None and out == stop_on:
             break
-        await asyncio.sleep(seconds)
+        delay = (
+            seconds[0] + (seconds[1] - seconds[0]) * random() if isinstance(seconds, tuple) else seconds  # nosec B311
+        )
+        await asyncio.sleep(delay)
 
 
 class BadWorkerState(RuntimeError):
@@ -78,16 +86,27 @@ def custom_exception_handler(loop: asyncio.AbstractEventLoop, context: dict[str,
         loop.set_exception_handler(custom_exception_handler)
         logging.info("Starting heartbeat.")
         loop.create_task(every(self.heartbeat, seconds=self.app_config.worker.heartbeat_interval_seconds))
-        loop.create_task(every(self.kill_zombies, seconds=self.app_config.worker.kill_zombies_interval_seconds))
+        loop.create_task(
+            every(
+                self.kill_zombies,
+                seconds=(
+                    self.app_config.worker.kill_zombies_interval_seconds * 0.5,
+                    self.app_config.worker.kill_zombies_interval_seconds * 1.5,
+                ),
+            )
+        )
         loop.create_task(
             every(
                 self.kill_long_job,
                 worker_loop_executor=worker_loop_executor,
-                seconds=self.app_config.worker.kill_long_job_interval_seconds,
+                seconds=(
+                    self.app_config.worker.kill_long_job_interval_seconds * 0.5,
+                    self.app_config.worker.kill_long_job_interval_seconds * 1.5,
+                ),
             )
         )
         loop.run_until_complete(
-            every(self.is_worker_alive, worker_loop_executor=worker_loop_executor, seconds=1, stop_on=False)
+            every(self.is_worker_alive, worker_loop_executor=worker_loop_executor, seconds=1.0, stop_on=False)
         )
         if exceptions:
             raise RuntimeError(f"Some async tasks failed: {exceptions}")


Test patch: