Repository: scoutapp/scout_apm_python

Issue:
Samplers thread race condition
The current samplers thread code has a number of race conditions and missing context manager usage of the lock that could manifest as broken behaviour. This could also end up failing in an async context as will be implemented in #285 .


Discussion:

Resolving patch:
diff --git a/src/scout_apm/core/samplers/__init__.py b/src/scout_apm/core/samplers/__init__.py
--- a/src/scout_apm/core/samplers/__init__.py
+++ b/src/scout_apm/core/samplers/__init__.py
@@ -1,54 +1,2 @@
 # coding=utf-8
 from __future__ import absolute_import, division, print_function, unicode_literals
-
-import datetime as dt
-import logging
-import threading
-from os import getpid
-from time import sleep
-
-from scout_apm.core.commands import ApplicationEvent
-from scout_apm.core.context import AgentContext
-from scout_apm.core.samplers.cpu import Cpu
-from scout_apm.core.samplers.memory import Memory
-
-logger = logging.getLogger(__name__)
-
-
-class Samplers(object):
-    _thread_lock = threading.Semaphore()
-
-    @classmethod
-    def ensure_running(cls):
-        if cls._thread_lock.acquire(False):
-            th = threading.Thread(target=Samplers.run_samplers)
-            th.daemon = True
-            th.start()
-            cls._thread_lock.release()
-
-    @classmethod
-    def run_samplers(cls):
-        logger.debug("Starting Samplers. Acquiring samplers lock.")
-        try:
-            if cls._thread_lock.acquire(True):
-                logger.debug("Acquired samplers lock.")
-                instances = [Cpu(), Memory()]
-
-                while True:
-                    for instance in instances:
-                        event_value = instance.run()
-                        if event_value is not None:
-                            event_type = (
-                                instance.metric_type() + "/" + instance.metric_name()
-                            )
-                            event = ApplicationEvent(
-                                event_value=event_value,
-                                event_type=event_type,
-                                timestamp=dt.datetime.utcnow(),
-                                source="Pid: " + str(getpid()),
-                            )
-                            AgentContext.socket().send(event)
-                    sleep(60)
-        finally:
-            logger.debug("Shutting down samplers thread.")
-            cls._thread_lock.release()
diff --git a/src/scout_apm/core/samplers/thread.py b/src/scout_apm/core/samplers/thread.py
new file mode 100644
--- /dev/null
+++ b/src/scout_apm/core/samplers/thread.py
@@ -0,0 +1,71 @@
+# coding=utf-8
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import datetime as dt
+import logging
+import os
+import threading
+
+from scout_apm.core.commands import ApplicationEvent
+from scout_apm.core.context import AgentContext
+from scout_apm.core.samplers.cpu import Cpu
+from scout_apm.core.samplers.memory import Memory
+
+logger = logging.getLogger(__name__)
+
+
+class SamplersThread(threading.Thread):
+    _instance = None
+    _instance_lock = threading.Lock()
+    _stop_event = threading.Event()
+
+    @classmethod
+    def ensure_started(cls):
+        with cls._instance_lock:
+            if cls._instance is None:
+                cls._instance = cls()
+                cls._instance.start()
+
+    @classmethod
+    def ensure_stopped(cls):
+        with cls._instance_lock:
+            if cls._instance is None:
+                # Nothing to stop
+                return
+            elif not cls._instance.is_alive():
+                # Thread died
+                cls._instance = None
+                return
+
+            # Signal stopping
+            cls._stop_event.set()
+            cls._instance.join()
+
+            cls._instance = None
+            cls._stop_event.clear()
+
+    def __init__(self, *args, **kwargs):
+        super(SamplersThread, self).__init__(*args, **kwargs)
+        self.daemon = True
+
+    def run(self):
+        logger.debug("Starting Samplers.")
+        instances = [Cpu(), Memory()]
+
+        while True:
+            for instance in instances:
+                event_value = instance.run()
+                if event_value is not None:
+                    event_type = instance.metric_type() + "/" + instance.metric_name()
+                    event = ApplicationEvent(
+                        event_value=event_value,
+                        event_type=event_type,
+                        timestamp=dt.datetime.utcnow(),
+                        source="Pid: " + str(os.getpid()),
+                    )
+                    AgentContext.socket().send(event)
+
+            should_stop = self._stop_event.wait(timeout=60)
+            if should_stop:
+                logger.debug("Stopping Samplers.")
+                break
diff --git a/src/scout_apm/core/tracked_request.py b/src/scout_apm/core/tracked_request.py
--- a/src/scout_apm/core/tracked_request.py
+++ b/src/scout_apm/core/tracked_request.py
@@ -9,7 +9,8 @@
 from scout_apm.core.commands import BatchCommand
 from scout_apm.core.context import AgentContext
 from scout_apm.core.n_plus_one_call_set import NPlusOneCallSet
-from scout_apm.core.samplers import Memory, Samplers
+from scout_apm.core.samplers.memory import Memory
+from scout_apm.core.samplers.thread import SamplersThread
 from scout_apm.core.thread_local import ThreadLocalSingleton
 
 logger = logging.getLogger(__name__)
@@ -115,7 +116,7 @@ def finish(self):
             if not self.is_ignored():
                 batch_command = BatchCommand.from_tracked_request(self)
                 AgentContext.socket().send(batch_command)
-            Samplers.ensure_running()
+            SamplersThread.ensure_started()
 
         # This can fail if the Tracked Request was created directly,
         # not through instance()


Test patch:
diff --git a/tests/unit/core/samplers/test_thread.py b/tests/unit/core/samplers/test_thread.py
new file mode 100644
--- /dev/null
+++ b/tests/unit/core/samplers/test_thread.py
@@ -0,0 +1,43 @@
+# coding=utf-8
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import time
+
+import pytest
+
+from scout_apm.core.samplers.thread import SamplersThread
+
+
+@pytest.fixture(autouse=True)
+def ensure_stopped_before_and_after():
+    SamplersThread.ensure_stopped()
+    try:
+        yield
+    finally:
+        assert not SamplersThread._stop_event.is_set()
+        SamplersThread.ensure_stopped()
+
+
+def test_ensure_started(caplog):
+    SamplersThread.ensure_started()
+    time.sleep(0.001)
+
+    assert SamplersThread._instance is not None
+    assert caplog.record_tuples[0] == (
+        "scout_apm.core.samplers.thread",
+        10,
+        "Starting Samplers.",
+    )
+
+
+def test_ensure_stopped(caplog):
+    SamplersThread.ensure_started()
+    time.sleep(0.001)
+    SamplersThread.ensure_stopped()
+
+    assert SamplersThread._instance is None
+    assert caplog.record_tuples[-1] == (
+        "scout_apm.core.samplers.thread",
+        10,
+        "Stopping Samplers.",
+    )