Repository: inventree/InvenTree

Issue:
[FR] Add tracing support
### Please verify that this feature request has NOT been suggested before.

- [X] I checked and didn't find a similar feature request

### Problem statement

It is hard to set up continous profiling / tracing without touching code. This makes usage of the official docker image unpractical in enterprise deployments.

### Suggested solution

Adding opentelemetry would aliviate this and is a vendor agnostic solution as it it a CNCF standard. The [adoption](https://opentelemetry.io/ecosystem/vendors/) is very wide reaching and consists of basically all popular APM plattform.

### Describe alternatives you've considered

Using sentry but running sentry on-prem can be a bit painful.

### Examples of other systems

_No response_

### Do you want to develop this?

- [X] I want to develop this.


Discussion:

Resolving patch:
diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py
--- a/InvenTree/InvenTree/settings.py
+++ b/InvenTree/InvenTree/settings.py
@@ -26,6 +26,7 @@
 
 from InvenTree.config import get_boolean_setting, get_custom_file, get_setting
 from InvenTree.sentry import default_sentry_dsn, init_sentry
+from InvenTree.tracing import setup_instruments, setup_tracing
 from InvenTree.version import checkMinPythonVersion, inventreeApiVersion
 
 from . import config, locales
@@ -709,6 +710,14 @@
     'INVENTREE_REMOTE_LOGIN_HEADER', 'remote_login_header', 'REMOTE_USER'
 )
 
+# region Tracing / error tracking
+inventree_tags = {
+    'testing': TESTING,
+    'docker': DOCKER,
+    'debug': DEBUG,
+    'remote': REMOTE_LOGIN,
+}
+
 # sentry.io integration for error reporting
 SENTRY_ENABLED = get_boolean_setting(
     'INVENTREE_SENTRY_ENABLED', 'sentry_enabled', False
@@ -721,15 +730,42 @@
 )
 
 if SENTRY_ENABLED and SENTRY_DSN:  # pragma: no cover
-    inventree_tags = {
-        'testing': TESTING,
-        'docker': DOCKER,
-        'debug': DEBUG,
-        'remote': REMOTE_LOGIN,
-    }
-
     init_sentry(SENTRY_DSN, SENTRY_SAMPLE_RATE, inventree_tags)
 
+# OpenTelemetry tracing
+TRACING_ENABLED = get_boolean_setting(
+    'INVENTREE_TRACING_ENABLED', 'tracing.enabled', False
+)
+if TRACING_ENABLED:  # pragma: no cover
+    _t_endpoint = get_setting('INVENTREE_TRACING_ENDPOINT', 'tracing.endpoint', None)
+    _t_headers = get_setting('INVENTREE_TRACING_HEADERS', 'tracing.headers', None, dict)
+    if _t_endpoint and _t_headers:
+        _t_resources = get_setting(
+            'INVENTREE_TRACING_RESOURCES', 'tracing.resources', {}, dict
+        )
+        cstm_tags = {'inventree.env.' + k: v for k, v in inventree_tags.items()}
+        tracing_resources = {**cstm_tags, **_t_resources}
+
+        setup_tracing(
+            _t_endpoint,
+            _t_headers,
+            tracing_resources,
+            get_boolean_setting('INVENTREE_TRACING_CONSOLE', 'tracing.console', False),
+            get_setting('INVENTREE_TRACING_AUTH', 'tracing.auth', {}),
+            get_setting('INVENTREE_TRACING_IS_HTTP', 'tracing.is_http', False),
+            get_boolean_setting(
+                'INVENTREE_TRACING_APPEND_HTTP', 'tracing.append_http', True
+            ),
+        )
+        # Run tracing/logging instrumentation
+        setup_instruments()
+    else:
+        logger.warning(
+            'OpenTelemetry tracing not enabled because endpoint or headers are not set'
+        )
+
+# endregion
+
 # Cache configuration
 cache_host = get_setting('INVENTREE_CACHE_HOST', 'cache.host', None)
 cache_port = get_setting('INVENTREE_CACHE_PORT', 'cache.port', '6379', typecast=int)
diff --git a/InvenTree/InvenTree/tracing.py b/InvenTree/InvenTree/tracing.py
new file mode 100644
--- /dev/null
+++ b/InvenTree/InvenTree/tracing.py
@@ -0,0 +1,142 @@
+"""OpenTelemetry setup functions."""
+
+import base64
+import logging
+from typing import Optional
+
+from opentelemetry import metrics, trace
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.redis import RedisInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk import _logs as logs
+from opentelemetry.sdk import resources
+from opentelemetry.sdk._logs import export as logs_export
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import (
+    ConsoleMetricExporter,
+    PeriodicExportingMetricReader,
+)
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+
+from InvenTree.version import inventreeVersion
+
+# Logger configuration
+logger = logging.getLogger('inventree')
+
+
+def setup_tracing(
+    endpoint: str,
+    headers: dict,
+    resources_input: Optional[dict] = None,
+    console: bool = False,
+    auth: Optional[dict] = None,
+    is_http: bool = False,
+    append_http: bool = True,
+):
+    """Set up tracing for the application in the current context.
+
+    Args:
+        endpoint: The endpoint to send the traces to.
+        headers: The headers to send with the traces.
+        resources_input: The resources to send with the traces.
+        console: Whether to output the traces to the console.
+    """
+    if resources_input is None:
+        resources_input = {}
+    if auth is None:
+        auth = {}
+
+    # Setup the auth headers
+    if 'basic' in auth:
+        basic_auth = auth['basic']
+        if 'username' in basic_auth and 'password' in basic_auth:
+            auth_raw = f'{basic_auth["username"]}:{basic_auth["password"]}'
+            auth_token = base64.b64encode(auth_raw.encode('utf-8')).decode('utf-8')
+            headers['Authorization'] = f'Basic {auth_token}'
+        else:
+            logger.warning('Basic auth is missing username or password')
+
+    # Clean up headers
+    headers = {k: v for k, v in headers.items() if v is not None}
+
+    # Initialize the OTLP Resource
+    resource = resources.Resource(
+        attributes={
+            resources.SERVICE_NAME: 'BACKEND',
+            resources.SERVICE_NAMESPACE: 'INVENTREE',
+            resources.SERVICE_VERSION: inventreeVersion(),
+            **resources_input,
+        }
+    )
+
+    # Import the OTLP exporters
+    if is_http:
+        from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+        from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
+            OTLPMetricExporter,
+        )
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+            OTLPSpanExporter,
+        )
+    else:
+        from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+        from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+            OTLPMetricExporter,
+        )
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+            OTLPSpanExporter,
+        )
+
+    # Spans / Tracs
+    span_exporter = OTLPSpanExporter(
+        headers=headers,
+        endpoint=endpoint if not (is_http and append_http) else f'{endpoint}/v1/traces',
+    )
+    trace_processor = BatchSpanProcessor(span_exporter)
+    trace_provider = TracerProvider(resource=resource)
+    trace.set_tracer_provider(trace_provider)
+    trace_provider.add_span_processor(trace_processor)
+    # For debugging purposes, export the traces to the console
+    if console:
+        trace_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+
+    # Metrics
+    metric_perodic_reader = PeriodicExportingMetricReader(
+        OTLPMetricExporter(
+            headers=headers,
+            endpoint=endpoint
+            if not (is_http and append_http)
+            else f'{endpoint}/v1/metrics',
+        )
+    )
+    metric_readers = [metric_perodic_reader]
+
+    # For debugging purposes, export the metrics to the console
+    if console:
+        console_metric_exporter = ConsoleMetricExporter()
+        console_metric_reader = PeriodicExportingMetricReader(console_metric_exporter)
+        metric_readers.append(console_metric_reader)
+
+    meter_provider = MeterProvider(resource=resource, metric_readers=metric_readers)
+    metrics.set_meter_provider(meter_provider)
+
+    # Logs
+    log_exporter = OTLPLogExporter(
+        headers=headers,
+        endpoint=endpoint if not (is_http and append_http) else f'{endpoint}/v1/logs',
+    )
+    log_provider = logs.LoggerProvider(resource=resource)
+    log_provider.add_log_record_processor(
+        logs_export.BatchLogRecordProcessor(log_exporter)
+    )
+    handler = logs.LoggingHandler(level=logging.INFO, logger_provider=log_provider)
+    logger = logging.getLogger('inventree')
+    logger.addHandler(handler)
+
+
+def setup_instruments():
+    """Run auto-insturmentation for OpenTelemetry tracing."""
+    DjangoInstrumentor().instrument()
+    RedisInstrumentor().instrument()
+    RequestsInstrumentor().instrument()
diff --git a/InvenTree/manage.py b/InvenTree/manage.py
--- a/InvenTree/manage.py
+++ b/InvenTree/manage.py
@@ -8,6 +8,7 @@
 def main():
     """Run administrative tasks."""
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'InvenTree.settings')
+
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:  # pragma: no cover


Test patch: