Repository: DataDog/dd-trace-py

Issue:
Django API APM traces disappeared after upgrade from 0.29.0 to 0.30.0
In talking with @brettlangdon we couldn't find a quick solution, but downgrading the library version brought them back. Interestingly enough, even the custom traces we were using within the API disappeared but all the ones that exist outside of Django i.e. workers, are all fine on version 0.30.0


Discussion:

Resolving patch:
diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py
--- a/ddtrace/internal/writer.py
+++ b/ddtrace/internal/writer.py
@@ -1,7 +1,6 @@
 # stdlib
 import itertools
 import random
-import os
 import time
 
 from .. import api
@@ -33,7 +32,7 @@ def __init__(self, hostname='localhost', port=8126, uds_path=None, https=False,
         super(AgentWriter, self).__init__(interval=self.QUEUE_PROCESSING_INTERVAL,
                                           exit_timeout=shutdown_timeout,
                                           name=self.__class__.__name__)
-        self._reset_queue()
+        self._trace_queue = Q(maxsize=MAX_TRACES)
         self._filters = filters
         self._priority_sampler = priority_sampler
         self._last_error_ts = 0
@@ -43,6 +42,23 @@ def __init__(self, hostname='localhost', port=8126, uds_path=None, https=False,
         self._stats_rate_counter = 0
         self.start()
 
+    def recreate(self):
+        """ Create a new instance of :class:`AgentWriter` using the same settings from this instance
+
+        :rtype: :class:`AgentWriter`
+        :returns: A new :class:`AgentWriter` instance
+        """
+        return self.__class__(
+            hostname=self.api.hostname,
+            port=self.api.port,
+            uds_path=self.api.uds_path,
+            https=self.api.https,
+            shutdown_timeout=self.exit_timeout,
+            filters=self._filters,
+            priority_sampler=self._priority_sampler,
+            dogstatsd=self.dogstatsd,
+        )
+
     def _send_stats(self):
         """Determine if we're sending stats or not.
 
@@ -62,18 +78,7 @@ def _send_stats(self):
 
         return False
 
-    def _reset_queue(self):
-        self._pid = os.getpid()
-        self._trace_queue = Q(maxsize=MAX_TRACES)
-
     def write(self, spans=None, services=None):
-        # if this queue was created in a different process (i.e. this was
-        # forked) reset everything so that we can safely work from it.
-        pid = os.getpid()
-        if self._pid != pid:
-            log.debug('resetting queues. pids(old:%s new:%s)', self._pid, pid)
-            self._reset_queue()
-
         if spans:
             self._trace_queue.put(spans)
 
diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py
--- a/ddtrace/tracer.py
+++ b/ddtrace/tracer.py
@@ -378,8 +378,7 @@ def start_span(self, name, child_of=None, service=None, resource=None, span_type
         context.add_span(span)
 
         # check for new process if runtime metrics worker has already been started
-        if self._runtime_worker:
-            self._check_new_process()
+        self._check_new_process()
 
         # update set of services handled by tracer
         if service and service not in self._services:
@@ -427,6 +426,9 @@ def _check_new_process(self):
         # and generated a new runtime id
         self._update_dogstatsd_constant_tags()
 
+        # Re-create the background writer thread
+        self.writer = self.writer.recreate()
+
     def trace(self, name, service=None, resource=None, span_type=None):
         """
         Return a span that will trace an operation called `name`. The context that created


Test patch:
diff --git a/tests/test_tracer.py b/tests/test_tracer.py
--- a/tests/test_tracer.py
+++ b/tests/test_tracer.py
@@ -1,7 +1,8 @@
 """
 tests for Tracer and utilities.
 """
-
+import contextlib
+import multiprocessing
 from os import getpid
 import sys
 import warnings
@@ -588,3 +589,63 @@ def test_tracer_dogstatsd_url():
     with pytest.raises(ValueError) as e:
         t = ddtrace.Tracer(dogstatsd_url='foo://foobar:12')
         assert str(e) == 'Unknown url format for `foo://foobar:12`'
+
+
+def test_tracer_fork():
+    t = ddtrace.Tracer()
+    original_pid = t._pid
+    original_writer = t.writer
+
+    @contextlib.contextmanager
+    def capture_failures(errors):
+        try:
+            yield
+        except AssertionError as e:
+            errors.put(e)
+
+    def task(t, errors):
+        # Start a new span to trigger process checking
+        with t.trace('test', service='test') as span:
+
+            # Assert we recreated the writer and have a new queue
+            with capture_failures(errors):
+                assert t._pid != original_pid
+                assert t.writer != original_writer
+                assert t.writer._trace_queue != original_writer._trace_queue
+
+            # Stop the background worker so we don't accidetnally flush the
+            # queue before we can assert on it
+            t.writer.stop()
+            t.writer.join()
+
+        # Assert the trace got written into the correct queue
+        assert original_writer._trace_queue.qsize() == 0
+        assert t.writer._trace_queue.qsize() == 1
+        assert [[span]] == list(t.writer._trace_queue.get())
+
+    # Assert tracer in a new process correctly recreates the writer
+    errors = multiprocessing.Queue()
+    p = multiprocessing.Process(target=task, args=(t, errors))
+    try:
+        p.start()
+    finally:
+        p.join(timeout=2)
+
+    while errors.qsize() > 0:
+        raise errors.get()
+
+    # Ensure writing into the tracer in this process still works as expected
+    with t.trace('test', service='test') as span:
+        assert t._pid == original_pid
+        assert t.writer == original_writer
+        assert t.writer._trace_queue == original_writer._trace_queue
+
+        # Stop the background worker so we don't accidentally flush the
+        # queue before we can assert on it
+        t.writer.stop()
+        t.writer.join()
+
+    # Assert the trace got written into the correct queue
+    assert original_writer._trace_queue.qsize() == 1
+    assert t.writer._trace_queue.qsize() == 1
+    assert [[span]] == list(t.writer._trace_queue.get())