Repository: open-telemetry/opentelemetry-python-contrib

Issue:
Redis asyncio clients not correctly traced
Currently the redis asyncio clients (Redis, Pipeline etc) are not being traced correctly. Every trace is a tiny fragment of the actual time, occurring around the start time of the redis command being called.

I narrowed this down to the instrumentor for redis not wrapping the async variants as expected (it uses the same sync approach that it uses for the sync redis - and as a result the span only traces the coroutine creation, but not the coroutine actually being awaited).

The result is a very short trace (< 1ms) at the start of the redis call that doesn't actually show the full time of the redis command.

I have fixed this in a follow up PR, which I will put up shortly and link.

**Describe your environment**
Python version: 3.11
redis-py version: 4.5.5
opentelemetry-instrumentation-redis version: current master or 0.39b0
OS: Ubuntu

**Steps to reproduce**

Here is an example script, which calls an async redis command, and an async redis pipeline.

For each async call:
- A parent span is started
- asyncio.sleep for 50 milliseconds (to create a time delta between the parent span and the actual redis span for visual clarity)
- call a redis command (debug-sleep) for 100 milliseconds

The data can then be visualised and the parent span compared to the redis-instrumentor generated span to see that the span doesn't encompass the full time of the command.

```
import asyncio

from opentelemetry.instrumentation.redis import RedisInstrumentor
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
from redis.asyncio import Redis



tracer_provider = TracerProvider(
    resource=Resource.create({
        "service.name": "my-app",
        "service.instance.id": "my-app-4",
    }),
)
tracer = tracer_provider.get_tracer(__name__)


async def call_redis_command() -> None:
    redis_pool = Redis(host="0.0.0.0", port=6379)
    with tracer.start_as_current_span("redis-command-wrapper"):
        await asyncio.sleep(0.05)
        await redis_pool.execute_command("DEBUG", "SLEEP", "0.1")

async def call_redis_pipeline() -> None:
    redis_pool = Redis(host="0.0.0.0", port=6379)
    with tracer.start_as_current_span("redis-pipeline-wrapper"):
        await asyncio.sleep(0.05)
        p = await redis_pool.pipeline()
        p.execute_command("DEBUG", "SLEEP", "0.1")
        await p.execute()

exporter = ConsoleSpanExporter()
span_processor = BatchSpanProcessor(span_exporter=exporter)
tracer_provider.add_span_processor(span_processor)

RedisInstrumentor().instrument(tracer_provider=tracer_provider)

asyncio.run(call_redis_command())
asyncio.run(call_redis_pipeline())
```

Below I have used jaeger to visualise the traces for ease of reading

**What is the expected behavior?**
Expecting to see 100 milliseconds of a redis command, wrapped by the 150 milliseconds parent span
![image](https://github.com/open-telemetry/opentelemetry-python-contrib/assets/66319645/2e822246-350e-4a8a-8731-941fbc83ece4)
(This is a screenshot from when I fix the code)


**What is the actual behavior?**
Actually seeing a really tiny redis command span (< 0.2ms) wrapped by the 150millisecond parent span
![image](https://github.com/open-telemetry/opentelemetry-python-contrib/assets/66319645/a7fe29e6-2fbe-4b1b-ba65-4fb9b1457f73)
(This is a screenshot from the results of the example provided script)



Discussion:

Resolving patch:
diff --git a/instrumentation/opentelemetry-instrumentation-redis/src/opentelemetry/instrumentation/redis/__init__.py b/instrumentation/opentelemetry-instrumentation-redis/src/opentelemetry/instrumentation/redis/__init__.py
--- a/instrumentation/opentelemetry-instrumentation-redis/src/opentelemetry/instrumentation/redis/__init__.py
+++ b/instrumentation/opentelemetry-instrumentation-redis/src/opentelemetry/instrumentation/redis/__init__.py
@@ -136,6 +136,43 @@ def _set_connection_attributes(span, conn):
         span.set_attribute(key, value)
 
 
+def _build_span_name(instance, cmd_args):
+    if len(cmd_args) > 0 and cmd_args[0]:
+        name = cmd_args[0]
+    else:
+        name = instance.connection_pool.connection_kwargs.get("db", 0)
+    return name
+
+
+def _build_span_meta_data_for_pipeline(instance):
+    try:
+        command_stack = (
+            instance.command_stack
+            if hasattr(instance, "command_stack")
+            else instance._command_stack
+        )
+
+        cmds = [
+            _format_command_args(c.args if hasattr(c, "args") else c[0])
+            for c in command_stack
+        ]
+        resource = "\n".join(cmds)
+
+        span_name = " ".join(
+            [
+                (c.args[0] if hasattr(c, "args") else c[0][0])
+                for c in command_stack
+            ]
+        )
+    except (AttributeError, IndexError):
+        command_stack = []
+        resource = ""
+        span_name = ""
+
+    return command_stack, resource, span_name
+
+
+# pylint: disable=R0915
 def _instrument(
     tracer,
     request_hook: _RequestHookT = None,
@@ -143,11 +180,8 @@ def _instrument(
 ):
     def _traced_execute_command(func, instance, args, kwargs):
         query = _format_command_args(args)
+        name = _build_span_name(instance, args)
 
-        if len(args) > 0 and args[0]:
-            name = args[0]
-        else:
-            name = instance.connection_pool.connection_kwargs.get("db", 0)
         with tracer.start_as_current_span(
             name, kind=trace.SpanKind.CLIENT
         ) as span:
@@ -163,31 +197,11 @@ def _traced_execute_command(func, instance, args, kwargs):
             return response
 
     def _traced_execute_pipeline(func, instance, args, kwargs):
-        try:
-            command_stack = (
-                instance.command_stack
-                if hasattr(instance, "command_stack")
-                else instance._command_stack
-            )
-
-            cmds = [
-                _format_command_args(
-                    c.args if hasattr(c, "args") else c[0],
-                )
-                for c in command_stack
-            ]
-            resource = "\n".join(cmds)
-
-            span_name = " ".join(
-                [
-                    (c.args[0] if hasattr(c, "args") else c[0][0])
-                    for c in command_stack
-                ]
-            )
-        except (AttributeError, IndexError):
-            command_stack = []
-            resource = ""
-            span_name = ""
+        (
+            command_stack,
+            resource,
+            span_name,
+        ) = _build_span_meta_data_for_pipeline(instance)
 
         with tracer.start_as_current_span(
             span_name, kind=trace.SpanKind.CLIENT
@@ -232,32 +246,72 @@ def _traced_execute_pipeline(func, instance, args, kwargs):
             "ClusterPipeline.execute",
             _traced_execute_pipeline,
         )
+
+    async def _async_traced_execute_command(func, instance, args, kwargs):
+        query = _format_command_args(args)
+        name = _build_span_name(instance, args)
+
+        with tracer.start_as_current_span(
+            name, kind=trace.SpanKind.CLIENT
+        ) as span:
+            if span.is_recording():
+                span.set_attribute(SpanAttributes.DB_STATEMENT, query)
+                _set_connection_attributes(span, instance)
+                span.set_attribute("db.redis.args_length", len(args))
+            if callable(request_hook):
+                request_hook(span, instance, args, kwargs)
+            response = await func(*args, **kwargs)
+            if callable(response_hook):
+                response_hook(span, instance, response)
+            return response
+
+    async def _async_traced_execute_pipeline(func, instance, args, kwargs):
+        (
+            command_stack,
+            resource,
+            span_name,
+        ) = _build_span_meta_data_for_pipeline(instance)
+
+        with tracer.start_as_current_span(
+            span_name, kind=trace.SpanKind.CLIENT
+        ) as span:
+            if span.is_recording():
+                span.set_attribute(SpanAttributes.DB_STATEMENT, resource)
+                _set_connection_attributes(span, instance)
+                span.set_attribute(
+                    "db.redis.pipeline_length", len(command_stack)
+                )
+            response = await func(*args, **kwargs)
+            if callable(response_hook):
+                response_hook(span, instance, response)
+            return response
+
     if redis.VERSION >= _REDIS_ASYNCIO_VERSION:
         wrap_function_wrapper(
             "redis.asyncio",
             f"{redis_class}.execute_command",
-            _traced_execute_command,
+            _async_traced_execute_command,
         )
         wrap_function_wrapper(
             "redis.asyncio.client",
             f"{pipeline_class}.execute",
-            _traced_execute_pipeline,
+            _async_traced_execute_pipeline,
         )
         wrap_function_wrapper(
             "redis.asyncio.client",
             f"{pipeline_class}.immediate_execute_command",
-            _traced_execute_command,
+            _async_traced_execute_command,
         )
     if redis.VERSION >= _REDIS_ASYNCIO_CLUSTER_VERSION:
         wrap_function_wrapper(
             "redis.asyncio.cluster",
             "RedisCluster.execute_command",
-            _traced_execute_command,
+            _async_traced_execute_command,
         )
         wrap_function_wrapper(
             "redis.asyncio.cluster",
             "ClusterPipeline.execute",
-            _traced_execute_pipeline,
+            _async_traced_execute_pipeline,
         )
 
 


Test patch:
diff --git a/instrumentation/opentelemetry-instrumentation-redis/tests/test_redis.py b/instrumentation/opentelemetry-instrumentation-redis/tests/test_redis.py
--- a/instrumentation/opentelemetry-instrumentation-redis/tests/test_redis.py
+++ b/instrumentation/opentelemetry-instrumentation-redis/tests/test_redis.py
@@ -11,9 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import asyncio
 from unittest import mock
 
 import redis
+import redis.asyncio
 
 from opentelemetry import trace
 from opentelemetry.instrumentation.redis import RedisInstrumentor
@@ -21,6 +23,24 @@
 from opentelemetry.trace import SpanKind
 
 
+class AsyncMock:
+    """A sufficient async mock implementation.
+
+    Python 3.7 doesn't have an inbuilt async mock class, so this is used.
+    """
+
+    def __init__(self):
+        self.mock = mock.Mock()
+
+    async def __call__(self, *args, **kwargs):
+        future = asyncio.Future()
+        future.set_result("random")
+        return future
+
+    def __getattr__(self, item):
+        return AsyncMock()
+
+
 class TestRedis(TestBase):
     def setUp(self):
         super().setUp()
@@ -87,6 +107,35 @@ def test_instrument_uninstrument(self):
         spans = self.memory_exporter.get_finished_spans()
         self.assertEqual(len(spans), 1)
 
+    def test_instrument_uninstrument_async_client_command(self):
+        redis_client = redis.asyncio.Redis()
+
+        with mock.patch.object(redis_client, "connection", AsyncMock()):
+            asyncio.run(redis_client.get("key"))
+
+        spans = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(spans), 1)
+        self.memory_exporter.clear()
+
+        # Test uninstrument
+        RedisInstrumentor().uninstrument()
+
+        with mock.patch.object(redis_client, "connection", AsyncMock()):
+            asyncio.run(redis_client.get("key"))
+
+        spans = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(spans), 0)
+        self.memory_exporter.clear()
+
+        # Test instrument again
+        RedisInstrumentor().instrument()
+
+        with mock.patch.object(redis_client, "connection", AsyncMock()):
+            asyncio.run(redis_client.get("key"))
+
+        spans = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(spans), 1)
+
     def test_response_hook(self):
         redis_client = redis.Redis()
         connection = redis.connection.Connection()
diff --git a/tests/opentelemetry-docker-tests/tests/redis/test_redis_functional.py b/tests/opentelemetry-docker-tests/tests/redis/test_redis_functional.py
--- a/tests/opentelemetry-docker-tests/tests/redis/test_redis_functional.py
+++ b/tests/opentelemetry-docker-tests/tests/redis/test_redis_functional.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import asyncio
+from time import time_ns
 
 import redis
 import redis.asyncio
@@ -318,6 +319,29 @@ def test_basics(self):
         )
         self.assertEqual(span.attributes.get("db.redis.args_length"), 2)
 
+    def test_execute_command_traced_full_time(self):
+        """Command should be traced for coroutine execution time, not creation time."""
+        coro_created_time = None
+        finish_time = None
+
+        async def pipeline_simple():
+            nonlocal coro_created_time
+            nonlocal finish_time
+
+            # delay coroutine creation from coroutine execution
+            coro = self.redis_client.get("foo")
+            coro_created_time = time_ns()
+            await coro
+            finish_time = time_ns()
+
+        async_call(pipeline_simple())
+
+        spans = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(spans), 1)
+        span = spans[0]
+        self.assertTrue(span.start_time > coro_created_time)
+        self.assertTrue(span.end_time < finish_time)
+
     def test_pipeline_traced(self):
         async def pipeline_simple():
             async with self.redis_client.pipeline(
@@ -340,6 +364,35 @@ async def pipeline_simple():
         )
         self.assertEqual(span.attributes.get("db.redis.pipeline_length"), 3)
 
+    def test_pipeline_traced_full_time(self):
+        """Command should be traced for coroutine execution time, not creation time."""
+        coro_created_time = None
+        finish_time = None
+
+        async def pipeline_simple():
+            async with self.redis_client.pipeline(
+                transaction=False
+            ) as pipeline:
+                nonlocal coro_created_time
+                nonlocal finish_time
+                pipeline.set("blah", 32)
+                pipeline.rpush("foo", "éé")
+                pipeline.hgetall("xxx")
+
+                # delay coroutine creation from coroutine execution
+                coro = pipeline.execute()
+                coro_created_time = time_ns()
+                await coro
+                finish_time = time_ns()
+
+        async_call(pipeline_simple())
+
+        spans = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(spans), 1)
+        span = spans[0]
+        self.assertTrue(span.start_time > coro_created_time)
+        self.assertTrue(span.end_time < finish_time)
+
     def test_pipeline_immediate(self):
         async def pipeline_immediate():
             async with self.redis_client.pipeline() as pipeline:
@@ -359,6 +412,33 @@ async def pipeline_immediate():
             span.attributes.get(SpanAttributes.DB_STATEMENT), "SET ? ?"
         )
 
+    def test_pipeline_immediate_traced_full_time(self):
+        """Command should be traced for coroutine execution time, not creation time."""
+        coro_created_time = None
+        finish_time = None
+
+        async def pipeline_simple():
+            async with self.redis_client.pipeline(
+                transaction=False
+            ) as pipeline:
+                nonlocal coro_created_time
+                nonlocal finish_time
+                pipeline.set("a", 1)
+
+                # delay coroutine creation from coroutine execution
+                coro = pipeline.immediate_execute_command("SET", "b", 2)
+                coro_created_time = time_ns()
+                await coro
+                finish_time = time_ns()
+
+        async_call(pipeline_simple())
+
+        spans = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(spans), 1)
+        span = spans[0]
+        self.assertTrue(span.start_time > coro_created_time)
+        self.assertTrue(span.end_time < finish_time)
+
     def test_parent(self):
         """Ensure OpenTelemetry works with redis."""
         ot_tracer = trace.get_tracer("redis_svc")
@@ -408,6 +488,29 @@ def test_basics(self):
         )
         self.assertEqual(span.attributes.get("db.redis.args_length"), 2)
 
+    def test_execute_command_traced_full_time(self):
+        """Command should be traced for coroutine execution time, not creation time."""
+        coro_created_time = None
+        finish_time = None
+
+        async def pipeline_simple():
+            nonlocal coro_created_time
+            nonlocal finish_time
+
+            # delay coroutine creation from coroutine execution
+            coro = self.redis_client.get("foo")
+            coro_created_time = time_ns()
+            await coro
+            finish_time = time_ns()
+
+        async_call(pipeline_simple())
+
+        spans = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(spans), 1)
+        span = spans[0]
+        self.assertTrue(span.start_time > coro_created_time)
+        self.assertTrue(span.end_time < finish_time)
+
     def test_pipeline_traced(self):
         async def pipeline_simple():
             async with self.redis_client.pipeline(
@@ -430,6 +533,35 @@ async def pipeline_simple():
         )
         self.assertEqual(span.attributes.get("db.redis.pipeline_length"), 3)
 
+    def test_pipeline_traced_full_time(self):
+        """Command should be traced for coroutine execution time, not creation time."""
+        coro_created_time = None
+        finish_time = None
+
+        async def pipeline_simple():
+            async with self.redis_client.pipeline(
+                transaction=False
+            ) as pipeline:
+                nonlocal coro_created_time
+                nonlocal finish_time
+                pipeline.set("blah", 32)
+                pipeline.rpush("foo", "éé")
+                pipeline.hgetall("xxx")
+
+                # delay coroutine creation from coroutine execution
+                coro = pipeline.execute()
+                coro_created_time = time_ns()
+                await coro
+                finish_time = time_ns()
+
+        async_call(pipeline_simple())
+
+        spans = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(spans), 1)
+        span = spans[0]
+        self.assertTrue(span.start_time > coro_created_time)
+        self.assertTrue(span.end_time < finish_time)
+
     def test_parent(self):
         """Ensure OpenTelemetry works with redis."""
         ot_tracer = trace.get_tracer("redis_svc")