Repository: open-telemetry/opentelemetry-python

Issue:
Switch unit tests from `unittest.mock` to SDK & in-memory exporter
See https://github.com/open-telemetry/opentelemetry-python/pull/290#issuecomment-558091283.
Currently tests are cumbersome to write and actually we probably don't want to test which API calls are made but what Spans would result in most cases. For this a SDK with in-memory exporter would be better than using `unittest.mock`.


Discussion:

Resolving patch:
diff --git a/ext/opentelemetry-ext-testutil/src/opentelemetry/ext/testutil/wsgitestutil.py b/ext/opentelemetry-ext-testutil/src/opentelemetry/ext/testutil/wsgitestutil.py
--- a/ext/opentelemetry-ext-testutil/src/opentelemetry/ext/testutil/wsgitestutil.py
+++ b/ext/opentelemetry-ext-testutil/src/opentelemetry/ext/testutil/wsgitestutil.py
@@ -1,32 +1,38 @@
 import io
 import unittest
-import unittest.mock as mock
 import wsgiref.util as wsgiref_util
+from importlib import reload
 
 from opentelemetry import trace as trace_api
+from opentelemetry.sdk.trace import TracerSource, export
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+    InMemorySpanExporter,
+)
+
+_MEMORY_EXPORTER = None
 
 
 class WsgiTestBase(unittest.TestCase):
-    def setUp(self):
-        self.span = mock.create_autospec(trace_api.Span, spec_set=True)
-        tracer = trace_api.Tracer()
-        self.get_tracer_patcher = mock.patch.object(
-            trace_api.TracerSource,
-            "get_tracer",
-            autospec=True,
-            spec_set=True,
-            return_value=tracer,
-        )
-        self.get_tracer_patcher.start()
-
-        self.start_span_patcher = mock.patch.object(
-            tracer,
-            "start_span",
-            autospec=True,
-            spec_set=True,
-            return_value=self.span,
+    @classmethod
+    def setUpClass(cls):
+        global _MEMORY_EXPORTER  # pylint:disable=global-statement
+        trace_api.set_preferred_tracer_source_implementation(
+            lambda T: TracerSource()
         )
-        self.start_span = self.start_span_patcher.start()
+        tracer_source = trace_api.tracer_source()
+        _MEMORY_EXPORTER = InMemorySpanExporter()
+        span_processor = export.SimpleExportSpanProcessor(_MEMORY_EXPORTER)
+        tracer_source.add_span_processor(span_processor)
+
+    @classmethod
+    def tearDownClass(cls):
+        reload(trace_api)
+
+    def setUp(self):
+
+        self.memory_exporter = _MEMORY_EXPORTER
+        self.memory_exporter.clear()
+
         self.write_buffer = io.BytesIO()
         self.write = self.write_buffer.write
 
@@ -37,10 +43,6 @@ def setUp(self):
         self.response_headers = None
         self.exc_info = None
 
-    def tearDown(self):
-        self.get_tracer_patcher.stop()
-        self.start_span_patcher.stop()
-
     def start_response(self, status, response_headers, exc_info=None):
         self.status = status
         self.response_headers = response_headers


Test patch:
diff --git a/ext/opentelemetry-ext-flask/tests/test_flask_integration.py b/ext/opentelemetry-ext-flask/tests/test_flask_integration.py
--- a/ext/opentelemetry-ext-flask/tests/test_flask_integration.py
+++ b/ext/opentelemetry-ext-flask/tests/test_flask_integration.py
@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import unittest
-from unittest import mock
 
 from flask import Flask
 from werkzeug.test import Client
@@ -24,18 +23,28 @@
 from opentelemetry.ext.testutil.wsgitestutil import WsgiTestBase
 
 
+def expected_attributes(override_attributes):
+    default_attributes = {
+        "component": "http",
+        "http.method": "GET",
+        "http.server_name": "localhost",
+        "http.scheme": "http",
+        "host.port": 80,
+        "http.host": "localhost",
+        "http.target": "/",
+        "http.flavor": "1.1",
+        "http.status_text": "OK",
+        "http.status_code": 200,
+    }
+    for key, val in override_attributes.items():
+        default_attributes[key] = val
+    return default_attributes
+
+
 class TestFlaskIntegration(WsgiTestBase):
     def setUp(self):
         super().setUp()
 
-        self.span_attrs = {}
-
-        def setspanattr(key, value):
-            self.assertIsInstance(key, str)
-            self.span_attrs[key] = value
-
-        self.span.set_attribute = setspanattr
-
         self.app = Flask(__name__)
 
         def hello_endpoint(helloid):
@@ -49,100 +58,57 @@ def hello_endpoint(helloid):
         self.client = Client(self.app, BaseResponse)
 
     def test_simple(self):
-        resp = self.client.get("/hello/123")
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual([b"Hello: 123"], list(resp.response))
-
-        self.start_span.assert_called_with(
-            "hello_endpoint",
-            trace_api.INVALID_SPAN_CONTEXT,
-            kind=trace_api.SpanKind.SERVER,
-            attributes={
-                "component": "http",
-                "http.method": "GET",
-                "http.server_name": "localhost",
-                "http.scheme": "http",
-                "host.port": 80,
-                "http.host": "localhost",
+        expected_attrs = expected_attributes(
+            {
                 "http.target": "/hello/123",
-                "http.flavor": "1.1",
                 "http.route": "/hello/<int:helloid>",
-            },
-            start_time=mock.ANY,
-        )
-
-        # TODO: Change this test to use the SDK, as mocking becomes painful
-
-        self.assertEqual(
-            self.span_attrs,
-            {"http.status_code": 200, "http.status_text": "OK"},
+            }
         )
+        resp = self.client.get("/hello/123")
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual([b"Hello: 123"], list(resp.response))
+        span_list = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(span_list), 1)
+        self.assertEqual(span_list[0].name, "hello_endpoint")
+        self.assertEqual(span_list[0].kind, trace_api.SpanKind.SERVER)
+        self.assertEqual(span_list[0].attributes, expected_attrs)
 
     def test_404(self):
-        resp = self.client.post("/bye")
-        self.assertEqual(404, resp.status_code)
-        resp.close()
-
-        self.start_span.assert_called_with(
-            "/bye",
-            trace_api.INVALID_SPAN_CONTEXT,
-            kind=trace_api.SpanKind.SERVER,
-            attributes={
-                "component": "http",
+        expected_attrs = expected_attributes(
+            {
                 "http.method": "POST",
-                "http.server_name": "localhost",
-                "http.scheme": "http",
-                "host.port": 80,
-                "http.host": "localhost",
                 "http.target": "/bye",
-                "http.flavor": "1.1",
-            },
-            start_time=mock.ANY,
+                "http.status_text": "NOT FOUND",
+                "http.status_code": 404,
+            }
         )
 
-        # Nope, this uses Tracer.use_span(end_on_exit)
-        #  self.assertEqual(1, self.span.end.call_count)
-        # TODO: Change this test to use the SDK, as mocking becomes painful
-
-        self.assertEqual(
-            self.span_attrs,
-            {"http.status_code": 404, "http.status_text": "NOT FOUND"},
-        )
-
-    def test_internal_error(self):
-        resp = self.client.get("/hello/500")
-        self.assertEqual(500, resp.status_code)
+        resp = self.client.post("/bye")
+        self.assertEqual(404, resp.status_code)
         resp.close()
+        span_list = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(span_list), 1)
+        self.assertEqual(span_list[0].name, "/bye")
+        self.assertEqual(span_list[0].kind, trace_api.SpanKind.SERVER)
+        self.assertEqual(span_list[0].attributes, expected_attrs)
 
-        self.start_span.assert_called_with(
-            "hello_endpoint",
-            trace_api.INVALID_SPAN_CONTEXT,
-            kind=trace_api.SpanKind.SERVER,
-            attributes={
-                "component": "http",
-                "http.method": "GET",
-                "http.server_name": "localhost",
-                "http.scheme": "http",
-                "host.port": 80,
-                "http.host": "localhost",
+    def test_internal_error(self):
+        expected_attrs = expected_attributes(
+            {
                 "http.target": "/hello/500",
-                "http.flavor": "1.1",
                 "http.route": "/hello/<int:helloid>",
-            },
-            start_time=mock.ANY,
-        )
-
-        # Nope, this uses Tracer.use_span(end_on_exit)
-        #  self.assertEqual(1, self.span.end.call_count)
-        # TODO: Change this test to use the SDK, as mocking becomes painful
-
-        self.assertEqual(
-            self.span_attrs,
-            {
-                "http.status_code": 500,
                 "http.status_text": "INTERNAL SERVER ERROR",
-            },
+                "http.status_code": 500,
+            }
         )
+        resp = self.client.get("/hello/500")
+        self.assertEqual(500, resp.status_code)
+        resp.close()
+        span_list = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(span_list), 1)
+        self.assertEqual(span_list[0].name, "hello_endpoint")
+        self.assertEqual(span_list[0].kind, trace_api.SpanKind.SERVER)
+        self.assertEqual(span_list[0].attributes, expected_attrs)
 
 
 if __name__ == "__main__":
diff --git a/ext/opentelemetry-ext-wsgi/tests/test_wsgi_middleware.py b/ext/opentelemetry-ext-wsgi/tests/test_wsgi_middleware.py
--- a/ext/opentelemetry-ext-wsgi/tests/test_wsgi_middleware.py
+++ b/ext/opentelemetry-ext-wsgi/tests/test_wsgi_middleware.py
@@ -78,10 +78,8 @@ def validate_response(self, response, error=None):
         while True:
             try:
                 value = next(response)
-                self.assertEqual(0, self.span.end.call_count)
                 self.assertEqual(value, b"*")
             except StopIteration:
-                self.span.end.assert_called_once_with()
                 break
 
         self.assertEqual(self.status, "200 OK")
@@ -95,12 +93,13 @@ def validate_response(self, response, error=None):
         else:
             self.assertIsNone(self.exc_info)
 
-        # Verify that start_span has been called
-        self.start_span.assert_called_with(
-            "/",
-            trace_api.INVALID_SPAN_CONTEXT,
-            kind=trace_api.SpanKind.SERVER,
-            attributes={
+        span_list = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(span_list), 1)
+        self.assertEqual(span_list[0].name, "/")
+        self.assertEqual(span_list[0].kind, trace_api.SpanKind.SERVER)
+        self.assertEqual(
+            span_list[0].attributes,
+            {
                 "component": "http",
                 "http.method": "GET",
                 "http.server_name": "127.0.0.1",
@@ -109,6 +108,8 @@ def validate_response(self, response, error=None):
                 "http.host": "127.0.0.1",
                 "http.flavor": "1.0",
                 "http.url": "http://127.0.0.1/",
+                "http.status_text": "OK",
+                "http.status_code": 200,
             },
         )