Repository: open-telemetry/opentelemetry-python-contrib

Issue:
Celery Instrumentation TypeError
**Describe your environment** 

```
celery = "5.3.4"
opentelemetry-sdk = "1.20"
opentelemetry-propagator-b3 = "1.20"
opentelemetry-exporter-jaeger = "1.20"
opentelemetry-instrumentation-django = "0.41b0"
opentelemetry-instrumentation-asgi = "0.41b0"
opentelemetry-instrumentation-celery = "0.41b0"
```

**Steps to reproduce**

```py
$ cat web2/__init__.py

import logging
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mycoolproject.settings")  # NOQA

from kombu import Queue
from celery import Celery, signals
from django.conf import settings
from opentelemetry.instrumentation.celery import CeleryInstrumentor
from opentelemetry.exporter.jaeger.thrift import JaegerExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider


logger = logging.getLogger(__name__)


DISABLE_TRACING = os.getenv('DISABLE_TRACING', False) == 'true'
TRACING_COLLECTOR_ENDPOINT = os.getenv('TRACING_COLLECTOR_ENDPOINT', 'jaeger-collector.monitoring')
TRACING_COLLECTOR_PORT = os.getenv('TRACING_COLLECTOR_PORT', '14268')


@signals.worker_process_init.connect(weak=False)
def worker_process_init(*args, **kwargs):
    init_celery_tracing()


def init_celery_tracing():
    CeleryInstrumentor().instrument()

    trace.set_tracer_provider(TracerProvider())

    if DISABLE_TRACING:
        span_processor = BatchSpanProcessor(ConsoleSpanExporter())
    else:
        jaeger_exporter = JaegerExporter(
            collector_endpoint=f'http://{TRACING_COLLECTOR_ENDPOINT}:{TRACING_COLLECTOR_PORT}/api/traces?format=jaeger.thrift',
        )
        span_processor = BatchSpanProcessor(jaeger_exporter)

    trace.get_tracer_provider().add_span_processor(span_processor)


# this stops celery from hijacking the django logging config
@signals.setup_logging.connect
def setup_celery_logging(**kwargs):
    pass


tasks_app = Celery('web', broker=settings.CELERY_BROKER_URL)
tasks_app.conf.task_queues = (
    Queue('default2', routing_key='task.#'),
)
tasks_app.conf.task_default_queue = 'default2'
tasks_app.conf.task_default_exchange_type = 'topic'
tasks_app.conf.task_default_routing_key = 'task.default2'
tasks_app.conf.result_backend = 'redis://redis:6379/0'


tasks_app.conf.update(
    task_routes=({
        'web2.tasks.itsatest': {
            'queue': 'default2',
            'routing_key': 'task.default2.itsatest',
        },

    },),
    imports=(
        'web2.tasks',
    ),
    worker_prefetch_multiplier=5,
    task_soft_time_limit=300,
    task_time_limit=360,
    worker_max_tasks_per_child=100,
    broker_transport_options={
        'confirm_publish': True
    }
)
```

```py
$ cat web2/tasks.py

from web2 import tasks_app


@tasks_app.task(ignore_result=True)
def itsatest():
    print('itsatest')
    return 'itsatest'
```

```sh
poetry run celery --app web2.tasks worker
```

```
from web2.tasks import *

itsatest.delay()
```

**What is the expected behavior?**

No errors when executing tasks

**What is the actual behavior?**

```
[2023-11-01 20:14:58 +0000] [416] [celery.worker.strategy] [INFO] Task web2.tasks.itsatest[50b55ccd-be08-4f5c-82ee-538b363498ec] received
[2023-11-01 20:14:58 +0000] [416] [celery.pool] [DEBUG] TaskPool: Apply <function fast_trace_task at 0xffff8c758a60> (args:('web2.tasks.itsatest', '50b55ccd-be08-4f5c-82ee-538b363498ec', {'lang': 'py', 'task': 'web2.tasks.itsatest', 'id': '50b55ccd-be08-4f5c-82ee-538b363498ec', 'shadow': None, 'eta': None, 'expires': None, 'group': None, 'group_index': None, 'retries': 0, 'timelimit': [None, None], 'root_id': '50b55ccd-be08-4f5c-82ee-538b363498ec', 'parent_id': None, 'argsrepr': '()', 'kwargsrepr': '{}', 'origin': 'gen1251@4676bacd79b5', 'ignore_result': True, 'stamped_headers': None, 'stamps': {}, 'properties': {'content_type': 'application/json', 'content_encoding': 'utf-8', 'application_headers': {'lang': 'py', 'task': 'web2.tasks.itsatest', 'id': '50b55ccd-be08-4f5c-82ee-538b363498ec', 'shadow': None, 'eta': None, 'expires': None, 'group': None, 'group_index': None, 'retries': 0, 'timelimit': [None, None], 'root_id': '50b55ccd-be08-4f5c-82ee-538b363498ec', 'parent_id': None, 'argsrepr': '()', 'kwargsrepr': '{}', 'origin': 'gen1251@4676bacd79b5', 'ignore_result': True, 'stamped_headers': None, 'stamps': {}}, 'delivery_mode':... kwargs:{})
[2023-11-01 20:14:58 +0000] [518] [opentelemetry.instrumentation.celery] [DEBUG] prerun signal start task_id=50b55ccd-be08-4f5c-82ee-538b363498ec
itsatest
[2023-11-01 20:14:58 +0000] [518] [celery.app.trace] [INFO] Task web2.tasks.itsatest[50b55ccd-be08-4f5c-82ee-538b363498ec] succeeded in 0.013544000015826896s: 'itsatest'
[2023-11-01 20:14:58 +0000] [518] [opentelemetry.instrumentation.celery] [DEBUG] postrun signal task_id=50b55ccd-be08-4f5c-82ee-538b363498ec
[2023-11-01 20:14:58 +0000] [518] [celery.utils.dispatch.signal] [ERROR] Signal handler <bound method CeleryInstrumentor._trace_postrun of <opentelemetry.instrumentation.celery.CeleryInstrumentor object at 0xffff44692110>> raised: TypeError("'NoneType' object is not subscriptable")
Traceback (most recent call last):
  File "/opt/.venv/web-tq7C0_9c-py3.10/lib/python3.10/site-packages/celery/utils/dispatch/signal.py", line 276, in send
    response = receiver(signal=self, sender=sender, **named)
  File "/opt/.venv/web-tq7C0_9c-py3.10/lib/python3.10/site-packages/opentelemetry/instrumentation/celery/__init__.py", line 195, in _trace_postrun
    self._record_histograms(task_id, labels)
  File "/opt/.venv/web-tq7C0_9c-py3.10/lib/python3.10/site-packages/opentelemetry/instrumentation/celery/__init__.py", line 309, in _record_histograms
    self.metrics["flower.task.runtime.seconds"].record(
TypeError: 'NoneType' object is not subscriptable
```

**Additional context**

It looks like this functionality was introduced fairly recently in #1679 by @shalevr and @Akochavi 

Celery Instrumentation TypeError
**Describe your environment** 

```
celery = "5.3.4"
opentelemetry-sdk = "1.20"
opentelemetry-propagator-b3 = "1.20"
opentelemetry-exporter-jaeger = "1.20"
opentelemetry-instrumentation-django = "0.41b0"
opentelemetry-instrumentation-asgi = "0.41b0"
opentelemetry-instrumentation-celery = "0.41b0"
```

**Steps to reproduce**

```py
$ cat web2/__init__.py

import logging
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mycoolproject.settings")  # NOQA

from kombu import Queue
from celery import Celery, signals
from django.conf import settings
from opentelemetry.instrumentation.celery import CeleryInstrumentor
from opentelemetry.exporter.jaeger.thrift import JaegerExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider


logger = logging.getLogger(__name__)


DISABLE_TRACING = os.getenv('DISABLE_TRACING', False) == 'true'
TRACING_COLLECTOR_ENDPOINT = os.getenv('TRACING_COLLECTOR_ENDPOINT', 'jaeger-collector.monitoring')
TRACING_COLLECTOR_PORT = os.getenv('TRACING_COLLECTOR_PORT', '14268')


@signals.worker_process_init.connect(weak=False)
def worker_process_init(*args, **kwargs):
    init_celery_tracing()


def init_celery_tracing():
    CeleryInstrumentor().instrument()

    trace.set_tracer_provider(TracerProvider())

    if DISABLE_TRACING:
        span_processor = BatchSpanProcessor(ConsoleSpanExporter())
    else:
        jaeger_exporter = JaegerExporter(
            collector_endpoint=f'http://{TRACING_COLLECTOR_ENDPOINT}:{TRACING_COLLECTOR_PORT}/api/traces?format=jaeger.thrift',
        )
        span_processor = BatchSpanProcessor(jaeger_exporter)

    trace.get_tracer_provider().add_span_processor(span_processor)


# this stops celery from hijacking the django logging config
@signals.setup_logging.connect
def setup_celery_logging(**kwargs):
    pass


tasks_app = Celery('web', broker=settings.CELERY_BROKER_URL)
tasks_app.conf.task_queues = (
    Queue('default2', routing_key='task.#'),
)
tasks_app.conf.task_default_queue = 'default2'
tasks_app.conf.task_default_exchange_type = 'topic'
tasks_app.conf.task_default_routing_key = 'task.default2'
tasks_app.conf.result_backend = 'redis://redis:6379/0'


tasks_app.conf.update(
    task_routes=({
        'web2.tasks.itsatest': {
            'queue': 'default2',
            'routing_key': 'task.default2.itsatest',
        },

    },),
    imports=(
        'web2.tasks',
    ),
    worker_prefetch_multiplier=5,
    task_soft_time_limit=300,
    task_time_limit=360,
    worker_max_tasks_per_child=100,
    broker_transport_options={
        'confirm_publish': True
    }
)
```

```py
$ cat web2/tasks.py

from web2 import tasks_app


@tasks_app.task(ignore_result=True)
def itsatest():
    print('itsatest')
    return 'itsatest'
```

```sh
poetry run celery --app web2.tasks worker
```

```
from web2.tasks import *

itsatest.delay()
```

**What is the expected behavior?**

No errors when executing tasks

**What is the actual behavior?**

```
[2023-11-01 20:14:58 +0000] [416] [celery.worker.strategy] [INFO] Task web2.tasks.itsatest[50b55ccd-be08-4f5c-82ee-538b363498ec] received
[2023-11-01 20:14:58 +0000] [416] [celery.pool] [DEBUG] TaskPool: Apply <function fast_trace_task at 0xffff8c758a60> (args:('web2.tasks.itsatest', '50b55ccd-be08-4f5c-82ee-538b363498ec', {'lang': 'py', 'task': 'web2.tasks.itsatest', 'id': '50b55ccd-be08-4f5c-82ee-538b363498ec', 'shadow': None, 'eta': None, 'expires': None, 'group': None, 'group_index': None, 'retries': 0, 'timelimit': [None, None], 'root_id': '50b55ccd-be08-4f5c-82ee-538b363498ec', 'parent_id': None, 'argsrepr': '()', 'kwargsrepr': '{}', 'origin': 'gen1251@4676bacd79b5', 'ignore_result': True, 'stamped_headers': None, 'stamps': {}, 'properties': {'content_type': 'application/json', 'content_encoding': 'utf-8', 'application_headers': {'lang': 'py', 'task': 'web2.tasks.itsatest', 'id': '50b55ccd-be08-4f5c-82ee-538b363498ec', 'shadow': None, 'eta': None, 'expires': None, 'group': None, 'group_index': None, 'retries': 0, 'timelimit': [None, None], 'root_id': '50b55ccd-be08-4f5c-82ee-538b363498ec', 'parent_id': None, 'argsrepr': '()', 'kwargsrepr': '{}', 'origin': 'gen1251@4676bacd79b5', 'ignore_result': True, 'stamped_headers': None, 'stamps': {}}, 'delivery_mode':... kwargs:{})
[2023-11-01 20:14:58 +0000] [518] [opentelemetry.instrumentation.celery] [DEBUG] prerun signal start task_id=50b55ccd-be08-4f5c-82ee-538b363498ec
itsatest
[2023-11-01 20:14:58 +0000] [518] [celery.app.trace] [INFO] Task web2.tasks.itsatest[50b55ccd-be08-4f5c-82ee-538b363498ec] succeeded in 0.013544000015826896s: 'itsatest'
[2023-11-01 20:14:58 +0000] [518] [opentelemetry.instrumentation.celery] [DEBUG] postrun signal task_id=50b55ccd-be08-4f5c-82ee-538b363498ec
[2023-11-01 20:14:58 +0000] [518] [celery.utils.dispatch.signal] [ERROR] Signal handler <bound method CeleryInstrumentor._trace_postrun of <opentelemetry.instrumentation.celery.CeleryInstrumentor object at 0xffff44692110>> raised: TypeError("'NoneType' object is not subscriptable")
Traceback (most recent call last):
  File "/opt/.venv/web-tq7C0_9c-py3.10/lib/python3.10/site-packages/celery/utils/dispatch/signal.py", line 276, in send
    response = receiver(signal=self, sender=sender, **named)
  File "/opt/.venv/web-tq7C0_9c-py3.10/lib/python3.10/site-packages/opentelemetry/instrumentation/celery/__init__.py", line 195, in _trace_postrun
    self._record_histograms(task_id, labels)
  File "/opt/.venv/web-tq7C0_9c-py3.10/lib/python3.10/site-packages/opentelemetry/instrumentation/celery/__init__.py", line 309, in _record_histograms
    self.metrics["flower.task.runtime.seconds"].record(
TypeError: 'NoneType' object is not subscriptable
```

**Additional context**

It looks like this functionality was introduced fairly recently in #1679 by @shalevr and @Akochavi 



Discussion:
Validated reproduction steps: I've been able to reproduce this behavior roughly following the steps above. Take note that python 3.10 is expected.
@dacox @josephmcasey  ~can you try using `worker_ready` signal instead?~ 

UPD: `worker_ready` is not the right signal to instrument Celery, unfortunately. I think this is a combination of 2 issues: 
 * double initialization with the default instrumentation setup
 * the fact that instance level vars are reset on `CeleryInstrumentor.__init__()`
 
See below.

It looks like with `worker_process_init` `CeleryInstrumentor.__init__()` is called again and that effectively resets `self.metrics` to `None`.
Do you think it is fine as a possible quick workaround for now? A proper one would be on the CeleryInstrumentor, most likely.

Note: I even have to check specifically for `otel.instrumentation.celery.CeleryInstrumentor().is_instrumented_by_opentelemetry` not to see warnings on instrumenting twice. Any idea how to overcome this related issue?
FYI: A workaround with adding a unique enough `dispatch_uid` to the `worker_process_init.connect()` call worked for my setup.
@mikek , apologies for the delay in response, events in my personal life required me to be away from the computer. I'm unable to provide my full implementation, but the error provided above was resolvable by instantiating and controlling the meter provider passed into the instrumentor.

```python
# Sets the global default meter provider
metrics.set_meter_provider(provider)

# Creates a meter from the global meter provider
meter = metrics.get_meter("meter.celery")
CeleryInstrumentor().instrument(tracer_provider=tracer_provider, meter_provider=meter)
```

I would concur with you assessment that this instrumentor has been done incorrectly and needs to be fixed, but it is good to know there are adequate workarounds.
https://github.com/open-telemetry/opentelemetry-python-contrib/blob/7c12ad9844ac179e3f6a493491707a9bafd06f6b/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py#L118


https://github.com/open-telemetry/opentelemetry-python-contrib/blob/7c12ad9844ac179e3f6a493491707a9bafd06f6b/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py#L143
@mikek @josephmcasey sorry for the delay in response!

I'm a little unclear on what the workaround you're describing is 😅 
might this be a problem with a similar cause to https://github.com/open-telemetry/opentelemetry-python-contrib/issues/1790?
@mikek @josephmcasey If either of you have a moment I'd really appreciate some more information on a workaround for this. My knowledge of OTEL isn't great and I'm struggling to follow the suggestions mentioned above.
@dacox @paddyobrien I've tried, but was not able to reproduce this with the original setup in this issue.
However, all I had to do in my setup (I am not able to provide it now) was this:
```diff
- @worker_process_init.connect(weak=False)
+ @worker_process_init.connect(weak=False, dispatch_uid=f"init_celery_tracing p:{os.getpid()} t:{threading.get_native_id()}")
 def init_celery_tracing(*args, **kwargs):
     ...
```
There might be a better way, of course.
Thank you @mikek unfortunately this didn't help prevent the error for me.
None of the above solutions worked for me. I was able to get it working by doing:

```python
meter_provider = MeterProvider(resource=resource)
celery_instrumentor = CeleryInstrumentor()
celery_instrumentor.create_celery_metrics(meter_provider.get_meter('meter.celery'))
celery_instrumentor.instrument()
```

Not sure if it's a safe approach, but the error is gone. I am on:
```
opentelemetry-api==1.23.0
opentelemetry-sdk==1.23.0
opentelemetry-instrumentation-celery==0.44b0
```

Let me know if that works for you, @paddyobrien.
Validated reproduction steps: I've been able to reproduce this behavior roughly following the steps above. Take note that python 3.10 is expected.
@dacox @josephmcasey  ~can you try using `worker_ready` signal instead?~ 

UPD: `worker_ready` is not the right signal to instrument Celery, unfortunately. I think this is a combination of 2 issues: 
 * double initialization with the default instrumentation setup
 * the fact that instance level vars are reset on `CeleryInstrumentor.__init__()`
 
See below.

It looks like with `worker_process_init` `CeleryInstrumentor.__init__()` is called again and that effectively resets `self.metrics` to `None`.
Do you think it is fine as a possible quick workaround for now? A proper one would be on the CeleryInstrumentor, most likely.

Note: I even have to check specifically for `otel.instrumentation.celery.CeleryInstrumentor().is_instrumented_by_opentelemetry` not to see warnings on instrumenting twice. Any idea how to overcome this related issue?
FYI: A workaround with adding a unique enough `dispatch_uid` to the `worker_process_init.connect()` call worked for my setup.
@mikek , apologies for the delay in response, events in my personal life required me to be away from the computer. I'm unable to provide my full implementation, but the error provided above was resolvable by instantiating and controlling the meter provider passed into the instrumentor.

```python
# Sets the global default meter provider
metrics.set_meter_provider(provider)

# Creates a meter from the global meter provider
meter = metrics.get_meter("meter.celery")
CeleryInstrumentor().instrument(tracer_provider=tracer_provider, meter_provider=meter)
```

I would concur with you assessment that this instrumentor has been done incorrectly and needs to be fixed, but it is good to know there are adequate workarounds.
https://github.com/open-telemetry/opentelemetry-python-contrib/blob/7c12ad9844ac179e3f6a493491707a9bafd06f6b/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py#L118


https://github.com/open-telemetry/opentelemetry-python-contrib/blob/7c12ad9844ac179e3f6a493491707a9bafd06f6b/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py#L143
@mikek @josephmcasey sorry for the delay in response!

I'm a little unclear on what the workaround you're describing is 😅 
might this be a problem with a similar cause to https://github.com/open-telemetry/opentelemetry-python-contrib/issues/1790?
@mikek @josephmcasey If either of you have a moment I'd really appreciate some more information on a workaround for this. My knowledge of OTEL isn't great and I'm struggling to follow the suggestions mentioned above.
@dacox @paddyobrien I've tried, but was not able to reproduce this with the original setup in this issue.
However, all I had to do in my setup (I am not able to provide it now) was this:
```diff
- @worker_process_init.connect(weak=False)
+ @worker_process_init.connect(weak=False, dispatch_uid=f"init_celery_tracing p:{os.getpid()} t:{threading.get_native_id()}")
 def init_celery_tracing(*args, **kwargs):
     ...
```
There might be a better way, of course.
Thank you @mikek unfortunately this didn't help prevent the error for me.
None of the above solutions worked for me. I was able to get it working by doing:

```python
meter_provider = MeterProvider(resource=resource)
celery_instrumentor = CeleryInstrumentor()
celery_instrumentor.create_celery_metrics(meter_provider.get_meter('meter.celery'))
celery_instrumentor.instrument()
```

Not sure if it's a safe approach, but the error is gone. I am on:
```
opentelemetry-api==1.23.0
opentelemetry-sdk==1.23.0
opentelemetry-instrumentation-celery==0.44b0
```

Let me know if that works for you, @paddyobrien.

Resolving patch:
diff --git a/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py b/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py
--- a/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py
+++ b/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py
@@ -113,10 +113,8 @@ def keys(self, carrier):
 
 
 class CeleryInstrumentor(BaseInstrumentor):
-    def __init__(self):
-        super().__init__()
-        self.metrics = None
-        self.task_id_to_start_time = {}
+    metrics = None
+    task_id_to_start_time = {}
 
     def instrumentation_dependencies(self) -> Collection[str]:
         return _instruments


Test patch:
diff --git a/instrumentation/opentelemetry-instrumentation-celery/tests/test_duplicate.py b/instrumentation/opentelemetry-instrumentation-celery/tests/test_duplicate.py
new file mode 100644
--- /dev/null
+++ b/instrumentation/opentelemetry-instrumentation-celery/tests/test_duplicate.py
@@ -0,0 +1,30 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from opentelemetry.instrumentation.celery import CeleryInstrumentor
+
+
+class TestUtils(unittest.TestCase):
+    def test_duplicate_instrumentaion(self):
+        first = CeleryInstrumentor()
+        first.instrument()
+        second = CeleryInstrumentor()
+        second.instrument()
+        CeleryInstrumentor().uninstrument()
+        self.assertIsNotNone(first.metrics)
+        self.assertIsNotNone(second.metrics)
+        self.assertEqual(first.task_id_to_start_time, {})
+        self.assertEqual(second.task_id_to_start_time, {})