Repository: DataDog/dd-trace-py

Issue:
tests.test_integration.TestWorkers testMethod=test_worker_http_error_logging randomly fails
```
__________________ TestWorkers.test_worker_http_error_logging __________________

self = <tests.test_integration.TestWorkers testMethod=test_worker_http_error_logging>

    def test_worker_http_error_logging(self):
        # Tests the logging http error logic
        tracer = self.tracer
        self.tracer.writer.api = FlawedAPI(Tracer.DEFAULT_HOSTNAME, Tracer.DEFAULT_PORT)
        tracer.trace('client.testing').finish()
    
        log = logging.getLogger("ddtrace.writer")
        log_handler = MockedLogHandler(level='DEBUG')
        log.addHandler(log_handler)
    
        # sleeping 1.01 secs to prevent writer from exiting before logging
        time.sleep(1.01)
        self._wait_thread_flush()
        assert tracer.writer._worker._last_error_ts < time.time()
    
        logged_errors = log_handler.messages['error']
>       eq_(len(logged_errors), 1)
E       AssertionError: 0 != 1
```

Full log at https://circleci.com/gh/DataDog/dd-trace-py/105520


Discussion:

Resolving patch:
diff --git a/ddtrace/writer.py b/ddtrace/writer.py
--- a/ddtrace/writer.py
+++ b/ddtrace/writer.py
@@ -7,6 +7,7 @@
 
 from . import api
 from .internal.logger import get_logger
+from ddtrace.vendor.six.moves.queue import Queue, Full, Empty
 
 log = get_logger(__name__)
 
@@ -33,7 +34,7 @@ def write(self, spans=None, services=None):
         self._reset_worker()
 
         if spans:
-            self._traces.add(spans)
+            self._traces.put(spans)
 
     def _reset_worker(self):
         # if this queue was created in a different process (i.e. this was
@@ -41,7 +42,7 @@ def _reset_worker(self):
         pid = os.getpid()
         if self._pid != pid:
             log.debug('resetting queues. pids(old:%s new:%s)', self._pid, pid)
-            self._traces = Q(max_size=MAX_TRACES)
+            self._traces = Q(maxsize=MAX_TRACES)
             self._worker = None
             self._pid = pid
 
@@ -57,6 +58,8 @@ def _reset_worker(self):
 
 class AsyncWorker(object):
 
+    QUEUE_PROCESSING_INTERVAL = 1
+
     def __init__(self, api, trace_queue, service_queue=None, shutdown_timeout=DEFAULT_TIMEOUT,
                  filters=None, priority_sampler=None):
         self._trace_queue = trace_queue
@@ -66,6 +69,7 @@ def __init__(self, api, trace_queue, service_queue=None, shutdown_timeout=DEFAUL
         self._filters = filters
         self._priority_sampler = priority_sampler
         self._last_error_ts = 0
+        self._run = True
         self.api = api
         self.start()
 
@@ -87,7 +91,7 @@ def stop(self):
         """
         with self._lock:
             if self._thread and self.is_alive():
-                self._trace_queue.close()
+                self._run = False
 
     def join(self, timeout=2):
         """
@@ -101,12 +105,9 @@ def _on_shutdown(self):
             if not self._thread:
                 return
 
-            # wait for in-flight queues to get traced.
-            time.sleep(0.1)
-            self._trace_queue.close()
+            self._run = False
 
-            size = self._trace_queue.size()
-            if size:
+            if self._trace_queue.qsize():
                 key = 'ctrl-break' if os.name == 'nt' else 'ctrl-c'
                 log.debug(
                     'Waiting %ss for traces to be sent. Hit %s to quit.',
@@ -114,42 +115,44 @@ def _on_shutdown(self):
                     key,
                 )
                 timeout = time.time() + self._shutdown_timeout
-                while time.time() < timeout and self._trace_queue.size():
+                while time.time() < timeout and self._trace_queue.qsize():
                     # FIXME[matt] replace with a queue join
                     time.sleep(0.05)
 
     def _target(self):
-        traces_response = None
-
-        while True:
-            traces = self._trace_queue.pop()
-            if traces:
+        while self._run or self._trace_queue.qsize() > 0:
+            # Set a timeout so we check for self._run once in a while
+            try:
+                traces = self._trace_queue.get(block=False)
+            except Empty:
+                pass
+            else:
                 # Before sending the traces, make them go through the
                 # filters
                 try:
                     traces = self._apply_filters(traces)
                 except Exception as err:
-                    log.error('error while filtering traces:{0}'.format(err))
-            if traces:
-                # If we have data, let's try to send it.
-                try:
-                    traces_response = self.api.send_traces(traces)
-                except Exception as err:
-                    log.error('cannot send spans to {1}:{2}: {0}'.format(err, self.api.hostname, self.api.port))
+                    log.error('error while filtering traces: {0}'.format(err))
 
-            if self._trace_queue.closed() and self._trace_queue.size() == 0:
-                # no traces and the queue is closed. our work is done
-                return
+                traces_response = None
+
+                if traces:
+                    # If we have data, let's try to send it.
+                    try:
+                        traces_response = self.api.send_traces(traces)
+                    except Exception as err:
+                        log.error('cannot send spans to {1}:{2}: {0}'.format(
+                            err, self.api.hostname, self.api.port))
 
-            if self._priority_sampler and traces_response:
-                result_traces_json = traces_response.get_json()
-                if result_traces_json and 'rate_by_service' in result_traces_json:
-                    self._priority_sampler.set_sample_rate_by_service(result_traces_json['rate_by_service'])
+                if self._priority_sampler and traces_response:
+                    result_traces_json = traces_response.get_json()
+                    if result_traces_json and 'rate_by_service' in result_traces_json:
+                        self._priority_sampler.set_sample_rate_by_service(result_traces_json['rate_by_service'])
 
-            self._log_error_status(traces_response, 'traces')
-            traces_response = None
+                self._log_error_status(traces_response, 'traces')
 
-            time.sleep(1)  # replace with a blocking pop.
+            # Do not send data more often than QUEUE_PROCESSING_INTERVAL seconds
+            time.sleep(self.QUEUE_PROCESSING_INTERVAL)
 
     def _log_error_status(self, response, response_name):
         if not isinstance(response, api.Response):
@@ -188,45 +191,22 @@ def _apply_filters(self, traces):
         return traces
 
 
-class Q(object):
+class Q(Queue):
     """
     Q is a threadsafe queue that let's you pop everything at once and
     will randomly overwrite elements when it's over the max size.
     """
-    def __init__(self, max_size=1000):
-        self._things = []
-        self._lock = threading.Lock()
-        self._max_size = max_size
-        self._closed = False
-
-    def size(self):
-        with self._lock:
-            return len(self._things)
-
-    def close(self):
-        with self._lock:
-            self._closed = True
-
-    def closed(self):
-        with self._lock:
-            return self._closed
-
-    def add(self, thing):
-        with self._lock:
-            if self._closed:
-                return False
-
-            if len(self._things) < self._max_size or self._max_size <= 0:
-                self._things.append(thing)
-                return True
-            else:
-                idx = random.randrange(0, len(self._things))
-                self._things[idx] = thing
-
-    def pop(self):
-        with self._lock:
-            if not self._things:
-                return None
-            things = self._things
-            self._things = []
-            return things
+    def put(self, item):
+        try:
+            # Cannot use super() here because Queue in Python2 is old style class
+            return Queue.put(self, item, block=False)
+        except Full:
+            # If the queue is full, replace a random item
+            with self.mutex:
+                idx = random.randrange(0, self._qsize())
+                self.queue[idx] = item
+
+    def _get(self):
+        things = self.queue
+        self._init(self.maxsize)
+        return things


Test patch:
diff --git a/tests/test_integration.py b/tests/test_integration.py
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -83,7 +83,7 @@ def _wait_thread_flush(self):
         Helper that waits for the thread flush
         """
         self.tracer.writer._worker.stop()
-        self.tracer.writer._worker.join()
+        self.tracer.writer._worker.join(None)
 
     def _get_endpoint_payload(self, calls, endpoint):
         """
@@ -159,8 +159,6 @@ def test_worker_http_error_logging(self):
         log_handler = MockedLogHandler(level='DEBUG')
         log.addHandler(log_handler)
 
-        # sleeping 1.01 secs to prevent writer from exiting before logging
-        time.sleep(1.01)
         self._wait_thread_flush()
         assert tracer.writer._worker._last_error_ts < time.time()
 
diff --git a/tests/test_writer.py b/tests/test_writer.py
--- a/tests/test_writer.py
+++ b/tests/test_writer.py
@@ -52,7 +52,7 @@ def setUp(self):
         self.traces = Q()
         self.services = Q()
         for i in range(N_TRACES):
-            self.traces.add([
+            self.traces.put([
                 Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None)
                 for j in range(7)
             ])
@@ -96,3 +96,14 @@ def test_filters_short_circuit(self):
         worker.join()
         self.assertEqual(len(self.api.traces), 0)
         self.assertEqual(filtr.filtered_traces, 0)
+
+
+def test_queue_full():
+    q = Q(maxsize=3)
+    q.put(1)
+    q.put(2)
+    q.put(3)
+    q.put(4)
+    assert (list(q.queue) == [1, 2, 4] or
+            list(q.queue) == [1, 4, 3] or
+            list(q.queue) == [4, 2, 3])