Repository: locustio/locust

Issue:
Load being distributed to missing workers too Version 2.7.4.dev14
When running Locust in a distributed mode in Kubernetes, if the pod restarts then the worker status gets changed to missing. I can see that  spawning distributes the users equally to all workers, but in this case it distributes to missing workers as well which leads to reduced spwaning rate.
Kindly help!!
I have made some changes by adding Kubernetes support natively, have shared the details in slack channel. kindly suggest on how we can take this further.


Discussion:
Hi! @mboutet is the person most with most insight into user distribution, but I think he'll need more to go on.

Perhaps you can write a failing unit test that shows your issue? Look here for inspiration, and run tox or pytest to execute it. https://github.com/locustio/locust/blob/master/locust/test/test_dispatch.py

(my opinion has always been that it is ok for locust to fail miserably if workers go missing in the middle of a test, because IMO that test is already invalid :)

Resolving patch:
diff --git a/locust/runners.py b/locust/runners.py
--- a/locust/runners.py
+++ b/locust/runners.py
@@ -59,6 +59,7 @@
 CPU_MONITOR_INTERVAL = 5.0
 HEARTBEAT_INTERVAL = 1
 HEARTBEAT_LIVENESS = 3
+HEARTBEAT_DEAD_INTERNAL = -60
 FALLBACK_INTERVAL = 5
 
 
@@ -845,7 +846,12 @@ def heartbeat_worker(self):
                 self.reset_connection()
                 continue
 
+            missing_clients_to_be_removed = []
             for client in self.clients.all:
+                # if clients goes missing for more than HEARTBEAT_DEAD_INTERNAL then add them to be removed list
+                if client.state == STATE_MISSING and client.heartbeat <= HEARTBEAT_DEAD_INTERNAL:
+                    missing_clients_to_be_removed.append(client.id)
+
                 if client.heartbeat < 0 and client.state != STATE_MISSING:
                     logger.info("Worker %s failed to send heartbeat, setting state to missing." % str(client.id))
                     client.state = STATE_MISSING
@@ -861,6 +867,17 @@ def heartbeat_worker(self):
                 else:
                     client.heartbeat -= 1
 
+            # if there are any missing clients to be removed then remove them and trigger rebalance.
+            if len(missing_clients_to_be_removed) > 0:
+                for to_remove_client_id in missing_clients_to_be_removed:
+                    if self.clients.get(to_remove_client_id) is not None:
+                        del self.clients[to_remove_client_id]
+                if self.state == STATE_RUNNING or self.state == STATE_SPAWNING:
+                    # _users_dispatcher is set to none so that during redistribution the dead clients are not picked, alternative is to call self.stop() before start
+                    self._users_dispatcher = None
+                    # trigger redistribution after missing cclient removal
+                    self.start(user_count=self.target_user_count, spawn_rate=self.spawn_rate)
+
     def reset_connection(self):
         logger.info("Reset connection to worker")
         try:


Test patch:
diff --git a/locust/test/test_runners.py b/locust/test/test_runners.py
--- a/locust/test/test_runners.py
+++ b/locust/test/test_runners.py
@@ -1969,6 +1969,96 @@ def my_task(self):
             self.assertEqual(0, master.worker_count)
             self.assertEqual(STATE_STOPPED, master.state, "All workers went missing but test didn't stop.")
 
+    @mock.patch("locust.runners.HEARTBEAT_INTERVAL", new=0.1)
+    @mock.patch("locust.runners.HEARTBEAT_DEAD_INTERNAL", new=-3)
+    def test_worker_missing_after_heartbeat_dead_interval(self):
+        class TestUser(User):
+            @task
+            def my_task(self):
+                gevent.sleep(600)
+
+        with mock.patch("locust.rpc.rpc.Server", mocked_rpc()) as server, patch_env(
+            "LOCUST_WAIT_FOR_WORKERS_REPORT_AFTER_RAMP_UP", "0.1"
+        ):
+            master = self.get_runner(user_classes=[TestUser])
+            server.mocked_send(Message("client_ready", __version__, "fake_client1"))
+            server.mocked_send(Message("client_ready", __version__, "fake_client2"))
+            server.mocked_send(Message("client_ready", __version__, "fake_client3"))
+
+            master.start(3, 3)
+            server.mocked_send(Message("spawning", None, "fake_client1"))
+            server.mocked_send(Message("spawning", None, "fake_client2"))
+            server.mocked_send(Message("spawning", None, "fake_client3"))
+
+            sleep(0.1)
+            server.mocked_send(
+                Message(
+                    "heartbeat",
+                    {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1},
+                    "fake_client1",
+                )
+            )
+            server.mocked_send(
+                Message(
+                    "heartbeat",
+                    {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1},
+                    "fake_client2",
+                )
+            )
+            server.mocked_send(
+                Message(
+                    "heartbeat",
+                    {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1},
+                    "fake_client3",
+                )
+            )
+
+            sleep(0.1)
+            # initially all workers are in active state
+            self.assertEqual(0, len(master.clients.missing))
+            self.assertEqual(3, master.worker_count)
+            server.mocked_send(
+                Message(
+                    "heartbeat",
+                    {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1},
+                    "fake_client1",
+                )
+            )
+
+            server.mocked_send(
+                Message(
+                    "heartbeat",
+                    {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1},
+                    "fake_client2",
+                )
+            )
+
+            sleep(0.6)
+            # 4 intervals are passed since all 3 heart beats all workers are in missing state
+            self.assertEqual(3, len(master.clients.missing))
+            self.assertEqual(0, master.worker_count)
+
+            server.mocked_send(
+                Message(
+                    "heartbeat",
+                    {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1},
+                    "fake_client1",
+                )
+            )
+
+            server.mocked_send(
+                Message(
+                    "heartbeat",
+                    {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1},
+                    "fake_client2",
+                )
+            )
+            sleep(0.2)
+            # hearbeat received from two workers so they are active, for fake_client3 HEARTBEAT_DEAD_INTERNAL has been breached, so it will be removed from worker list
+            self.assertEqual(0, len(master.clients.missing))
+            self.assertEqual(2, master.worker_count)
+            master.stop()
+
     def test_master_total_stats(self):
         with mock.patch("locust.rpc.rpc.Server", mocked_rpc()) as server:
             master = self.get_runner()