Repository: DataDog/dd-trace-py

Issue:
fix(profiling): fix potential deadlock after fork()
When a process fork, its threading.Lock (and anything based on it) must be
considered unusable since there's no way to know in which state they are.

The workflow after fork()ing is to stop the old-parent-profiler, and start a
new one in the child.

The problem is that stopping the old profiler in the child would use at least
two locks:

1. one from the Event in the Scheduler that is used to stop the thread. This is
   fixed by checking if the thread actually needs to be stopped — which isn't the
   case in the child.
2. if a child process would try to release a Lock before before the fork, it
   could hit a deadlock from the Recorder's lock:
   When theading.Lock is wrapped by the Lock profiler, there's a small chance
   that it'll write an Event in the old-parent-Recorder. If the process forked
   while this Recorder lock was held by another thread, then trying to release
   a lock locked before the fork would dead lock:

     Traceback (most recent call first):
     Waiting for the GIL
     File "/src/ddtrace/profiling/recorder.py", line 56, in push_events
       with self._events_lock:
     File "/src/ddtrace/profiling/recorder.py", line 44, in push_event
       return self.push_events([event])
     File "/src/ddtrace/profiling/collector/threading.py", line 126, in release
       sampling_pct=self._self_capture_sampler.capture_pct,
     File "tests/profiling/simple_program_fork.py", line 33, in <module>
       lock_acquired_in_the_parent.release()



Discussion:
Feel free to run it 10 times to check. So far locally I was not able to reproduce the bug 🥳 
@jd seems like a different test failed 😕

ERROR:   py37-profile-gevent: commands failed

```
_________________ test_memory_collector_ignore_profiler[False] _________________

ignore_profiler = False

    @pytest.mark.parametrize(
        "ignore_profiler", (True, False),
    )
    def test_memory_collector_ignore_profiler(ignore_profiler):
        r = recorder.Recorder()
        mc = memalloc.MemoryCollector(r, ignore_profiler=ignore_profiler)
        with mc:
            _allocate_1k()
            # Make sure we collect at least once
            mc.periodic()
    
        ok = False
        for event in r.events[memalloc.MemoryAllocSampleEvent]:
            for frame in event.frames:
                if ignore_profiler:
                    assert frame[0] != _periodic.__file__
                elif frame[0] == _periodic.__file__:
                    ok = True
    
        if not ignore_profiler:
>           assert ok
E           assert False

tests/profiling/collector/test_memalloc.py:185: AssertionError
```
Re-running had it fail the same test for py38-profile-minreqs. Seems to be flaky from #1673 
Thanks @Kyle-Verhoog I'll take a look!

Resolving patch:
diff --git a/ddtrace/profiling/_periodic.py b/ddtrace/profiling/_periodic.py
--- a/ddtrace/profiling/_periodic.py
+++ b/ddtrace/profiling/_periodic.py
@@ -35,7 +35,12 @@ def __init__(self, interval, target, name=None, on_shutdown=None):
 
     def stop(self):
         """Stop the thread."""
-        self.quit.set()
+        # NOTE: make sure the thread is alive before using self.quit:
+        # 1. self.quit is Lock-based
+        # 2. if we're a child trying to stop a Thread,
+        #    the Lock might have been locked in a parent process while forking so that'd block forever
+        if self.is_alive():
+            self.quit.set()
 
     def run(self):
         """Run the target function periodically."""
diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py
--- a/ddtrace/profiling/profiler.py
+++ b/ddtrace/profiling/profiler.py
@@ -161,17 +161,17 @@ def stop(self, flush=True):
 
         :param flush: Wait for the flush of the remaining events before stopping.
         """
+        if self._scheduler:
+            self._scheduler.stop()
+
         for col in reversed(self._collectors):
             col.stop()
 
         for col in reversed(self._collectors):
             col.join()
 
-        if self._scheduler:
-            self._scheduler.stop()
-
-            if flush:
-                self._scheduler.join()
+        if self._scheduler and flush:
+            self._scheduler.join()
 
         self.status = ProfilerStatus.STOPPED
 
diff --git a/ddtrace/profiling/recorder.py b/ddtrace/profiling/recorder.py
--- a/ddtrace/profiling/recorder.py
+++ b/ddtrace/profiling/recorder.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf-8 -*-
 import collections
+import os
 
 from ddtrace.profiling import _nogevent
 from ddtrace.vendor import attr
@@ -32,6 +33,7 @@ class Recorder(object):
 
     events = attr.ib(init=False, repr=False)
     _events_lock = attr.ib(init=False, repr=False, factory=_nogevent.DoubleLock)
+    _pid = attr.ib(init=False, repr=False, factory=os.getpid)
 
     def __attrs_post_init__(self):
         self._reset_events()
@@ -51,7 +53,11 @@ def push_events(self, events):
 
         :param events: The event list to push.
         """
-        if events:
+        # NOTE: do not try to push events if the current PID has changed
+        # This means:
+        # 1. the process has forked
+        # 2. we don't know the state of _events_lock and it might be unusable — we'd deadlock
+        if events and os.getpid() == self._pid:
             event_type = events[0].__class__
             with self._events_lock:
                 q = self.events[event_type]


Test patch:
diff --git a/tests/profiling/collector/test_memalloc.py b/tests/profiling/collector/test_memalloc.py
--- a/tests/profiling/collector/test_memalloc.py
+++ b/tests/profiling/collector/test_memalloc.py
@@ -169,7 +169,7 @@ def test_memory_collector_ignore_profiler(ignore_profiler):
     r = recorder.Recorder()
     mc = memalloc.MemoryCollector(r, ignore_profiler=ignore_profiler)
     with mc:
-        _allocate_1k()
+        object()
         # Make sure we collect at least once
         mc.periodic()
 
@@ -180,6 +180,7 @@ def test_memory_collector_ignore_profiler(ignore_profiler):
                 assert frame[0] != _periodic.__file__
             elif frame[0] == _periodic.__file__:
                 ok = True
+                break
 
     if not ignore_profiler:
         assert ok
diff --git a/tests/profiling/test_main.py b/tests/profiling/test_main.py
--- a/tests/profiling/test_main.py
+++ b/tests/profiling/test_main.py
@@ -63,7 +63,6 @@ def test_call_script_pprof_output_interval(tmp_path, monkeypatch):
         check_pprof_file(filename + "." + str(pid) + (".%d" % i))
 
 
-@pytest.mark.skipif(os.getenv("DD_PROFILE_TEST_GEVENT", False), reason="Test hangs forever on gevent")
 def test_fork(tmp_path, monkeypatch):
     filename = str(tmp_path / "pprof")
     monkeypatch.setenv("DD_PROFILING_API_TIMEOUT", "0.1")