Repository: open-telemetry/opentelemetry-python-contrib

Issue:
Tornado instrumentation closes spans before on_finish is completed.
Ideally all trace spans in a request response cycle should be contained in one span. However because the tornado instrumentation library closes the span before calling the on_finish method. Orphaned spans from any instrumented code in on_finish method are created. Looking at the [code](https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py#L210-L212) a reordering would fix this situation. 

**Describe your environment** 

Python 3.8.5

requirements.txt
tornado==6.0.3

#tracing
opentelemetry-api==1.1.0
opentelemetry-sdk==1.1.0
opentelemetry-exporter-zipkin-json
opentelemetry-instrumentation
opentelemetry-instrumentation-dbapi
opentelemetry-instrumentation-tornado
opentelemetry-exporter-gcp-trace==1.0.0rc0



**Steps to reproduce**
Run the attached app below: 

`

import tornado.ioloop
import tornado.web
from opentelemetry import trace
from opentelemetry.trace import SpanKind

from opentelemetry import trace
from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter
from opentelemetry.exporter.zipkin.json import ZipkinExporter
from opentelemetry.sdk.resources import SERVICE_NAME, Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import (
    ConsoleSpanExporter,
    BatchSpanProcessor
)
from opentelemetry.instrumentation.tornado import TornadoInstrumentor


class MainHandler(tornado.web.RequestHandler):

    def on_finish(self) -> None:
        with trace.get_tracer(__name__).start_as_current_span("On finish test", kind=SpanKind.CLIENT) as span:
            print("Testing on finish span")

        super(MainHandler, self).on_finish()

    def get(self):
        self.write("Hello, world")




def init_tracing(is_deployment):
    resource = Resource.create(attributes={SERVICE_NAME: "Profile Service"})
    tracer_provider = TracerProvider(resource=resource)
    trace.set_tracer_provider(tracer_provider)

    if is_deployment:
        span_exporter = CloudTraceSpanExporter()
    else:
        # span_exporter = ZipkinExporter()
        span_exporter = ConsoleSpanExporter()

    span_processor = BatchSpanProcessor(span_exporter=span_exporter)
    trace.get_tracer_provider().add_span_processor(span_processor)

    TornadoInstrumentor().instrument()


def make_app():
    return tornado.web.Application([
        (r"/", MainHandler),
    ])


if __name__ == "__main__":

    init_tracing(is_deployment=False)
    app = make_app()
    app.listen(8888)
    tornado.ioloop.IOLoop.current().start()
`




**What is the expected behavior?**
All traces in a request cycle should be contained in one span. However because the tornado instrumentation library closes the span before calling the on_finish method. This causes an orphaned span to be created.

**What is the actual behavior?**
Tornado instrumentation library should possibly run on_finish method before closing the main trace span

**Additional context**
I looked at the code and saw the affected part is [here ](https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py#L210-L212). Is there any reason to close the span before the parent on_finish method can be run?



Discussion:

Resolving patch:
diff --git a/instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py b/instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py
--- a/instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py
+++ b/instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py
@@ -213,8 +213,9 @@ def _prepare(tracer, request_hook, func, handler, args, kwargs):
 
 
 def _on_finish(tracer, func, handler, args, kwargs):
+    response = func(*args, **kwargs)
     _finish_span(tracer, handler)
-    return func(*args, **kwargs)
+    return response
 
 
 def _log_exception(tracer, func, handler, args, kwargs):


Test patch:
diff --git a/instrumentation/opentelemetry-instrumentation-tornado/tests/test_instrumentation.py b/instrumentation/opentelemetry-instrumentation-tornado/tests/test_instrumentation.py
--- a/instrumentation/opentelemetry-instrumentation-tornado/tests/test_instrumentation.py
+++ b/instrumentation/opentelemetry-instrumentation-tornado/tests/test_instrumentation.py
@@ -347,6 +347,54 @@ def test_dynamic_handler(self):
             },
         )
 
+    def test_handler_on_finish(self):
+
+        response = self.fetch("/on_finish")
+        self.assertEqual(response.code, 200)
+
+        spans = self.sorted_spans(self.memory_exporter.get_finished_spans())
+        self.assertEqual(len(spans), 3)
+        auditor, server, client = spans
+
+        self.assertEqual(server.name, "FinishedHandler.get")
+        self.assertTrue(server.parent.is_remote)
+        self.assertNotEqual(server.parent, client.context)
+        self.assertEqual(server.parent.span_id, client.context.span_id)
+        self.assertEqual(server.context.trace_id, client.context.trace_id)
+        self.assertEqual(server.kind, SpanKind.SERVER)
+        self.assert_span_has_attributes(
+            server,
+            {
+                SpanAttributes.HTTP_METHOD: "GET",
+                SpanAttributes.HTTP_SCHEME: "http",
+                SpanAttributes.HTTP_HOST: "127.0.0.1:"
+                + str(self.get_http_port()),
+                SpanAttributes.HTTP_TARGET: "/on_finish",
+                SpanAttributes.NET_PEER_IP: "127.0.0.1",
+                SpanAttributes.HTTP_STATUS_CODE: 200,
+            },
+        )
+
+        self.assertEqual(client.name, "GET")
+        self.assertFalse(client.context.is_remote)
+        self.assertIsNone(client.parent)
+        self.assertEqual(client.kind, SpanKind.CLIENT)
+        self.assert_span_has_attributes(
+            client,
+            {
+                SpanAttributes.HTTP_URL: self.get_url("/on_finish"),
+                SpanAttributes.HTTP_METHOD: "GET",
+                SpanAttributes.HTTP_STATUS_CODE: 200,
+            },
+        )
+
+        self.assertEqual(auditor.name, "audit_task")
+        self.assertFalse(auditor.context.is_remote)
+        self.assertEqual(auditor.parent.span_id, server.context.span_id)
+        self.assertEqual(auditor.context.trace_id, client.context.trace_id)
+
+        self.assertEqual(auditor.kind, SpanKind.INTERNAL)
+
     def test_exclude_lists(self):
         def test_excluded(path):
             self.fetch(path)
diff --git a/instrumentation/opentelemetry-instrumentation-tornado/tests/tornado_test_app.py b/instrumentation/opentelemetry-instrumentation-tornado/tests/tornado_test_app.py
--- a/instrumentation/opentelemetry-instrumentation-tornado/tests/tornado_test_app.py
+++ b/instrumentation/opentelemetry-instrumentation-tornado/tests/tornado_test_app.py
@@ -1,4 +1,5 @@
 # pylint: disable=W0223,R0201
+import time
 
 import tornado.web
 from tornado import gen
@@ -79,6 +80,16 @@ def get(self):
         self.set_status(202)
 
 
+class FinishedHandler(tornado.web.RequestHandler):
+    def on_finish(self):
+        with self.application.tracer.start_as_current_span("audit_task"):
+            time.sleep(0.05)
+
+    def get(self):
+        self.write("Test can finish")
+        self.set_status(200)
+
+
 class HealthCheckHandler(tornado.web.RequestHandler):
     def get(self):
         self.set_status(200)
@@ -91,6 +102,7 @@ def make_app(tracer):
             (r"/error", BadHandler),
             (r"/cor", CoroutineHandler),
             (r"/async", AsyncHandler),
+            (r"/on_finish", FinishedHandler),
             (r"/healthz", HealthCheckHandler),
             (r"/ping", HealthCheckHandler),
         ]