Repository: microsoft/AzureTRE

Issue:
Application insights configuration for API missing extension configuration
When viewing the Application Insights blade in the portal for the API web app, indication is given that the integration is not fully enabled.

![image](https://user-images.githubusercontent.com/166592/123794341-03ed6f00-d8e3-11eb-9e09-32a2e5dec51b.png)

The instrumentation key is set why some data is sent to the logs but the full integration is not enabled. Manually enabling the integration from the portal adds additional configuration keys and values to the web app config.

Add missing config keys and values to enable the full integration.
`APPLICATIONINSIGHTS_CONNECTION_STRING`
`ApplicationInsightsAgent_EXTENSION_VERSION ~3`
`XDT_MicrosoftApplicationInsights_BaseExtensions`
et al.

[Reference](https://docs.microsoft.com/en-us/azure/azure-monitor/app/resource-manager-web-app)





Discussion:

Resolving patch:
diff --git a/management_api_app/main.py b/management_api_app/main.py
--- a/management_api_app/main.py
+++ b/management_api_app/main.py
@@ -4,7 +4,6 @@
 from fastapi import FastAPI
 from fastapi.exceptions import RequestValidationError
 from fastapi_utils.tasks import repeat_every
-from opencensus.ext.azure.log_exporter import AzureLogHandler
 from starlette.exceptions import HTTPException
 from starlette.middleware.errors import ServerErrorMiddleware
 
@@ -14,6 +13,7 @@
 from api.errors.generic_error import generic_error_handler
 from core import config
 from core.events import create_start_app_handler, create_stop_app_handler
+from services.logging import disable_unwanted_loggers, initialize_logging
 from service_bus.deployment_status_update import receive_message_and_update_deployment
 
 
@@ -40,46 +40,6 @@ def get_application() -> FastAPI:
     return application
 
 
-def initialize_logging(logging_level: int):
-    """
-    Adds the Application Insights handler for the root logger and sets the given logging level.
-
-    :param logging_level: The logging level to set e.g., logging.WARNING.
-    """
-    logger = logging.getLogger()
-
-    logging.getLogger("azure.core.pipeline.policies.http_logging_policy").disabled = True
-
-    logging.getLogger("azure.eventhub._eventprocessor.event_processor").disabled = True
-
-    logging.getLogger("azure.identity.aio._credentials.managed_identity").disabled = True
-    logging.getLogger("azure.identity.aio._credentials.environment").disabled = True
-    logging.getLogger("azure.identity.aio._internal.get_token_mixin").disabled = True
-    logging.getLogger("azure.identity.aio._internal.decorators").disabled = True
-    logging.getLogger("azure.identity.aio._credentials.chained").disabled = True
-    logging.getLogger("azure.identity").disabled = True
-
-    logging.getLogger("msal.token_cache").disabled = True
-
-    logging.getLogger("uamqp").disabled = True
-    logging.getLogger("uamqp.authentication.cbs_auth_async").disabled = True
-    logging.getLogger("uamqp.async_ops.client_async").disabled = True
-    logging.getLogger("uamqp.async_ops.connection_async").disabled = True
-    logging.getLogger("uamqp.async_ops").disabled = True
-    logging.getLogger("uamqp.authentication").disabled = True
-    logging.getLogger("uamqp.c_uamqp").disabled = True
-    logging.getLogger("uamqp.connection").disabled = True
-    logging.getLogger("uamqp.receiver").disabled = True
-
-    try:
-        logger.addHandler(AzureLogHandler(connection_string=f"InstrumentationKey={config.APP_INSIGHTS_INSTRUMENTATION_KEY}"))
-    except ValueError:
-        logger.error("Application Insights instrumentation key missing or invalid")
-
-    logging.basicConfig(level=logging_level)
-    logger.setLevel(logging_level)
-
-
 app = get_application()
 
 
@@ -90,6 +50,8 @@ async def initialize_logging_on_startup():
     else:
         initialize_logging(logging.INFO)
 
+    disable_unwanted_loggers()
+
 
 @app.on_event("startup")
 @repeat_every(seconds=20, wait_first=True, logger=logging.getLogger())
diff --git a/management_api_app/services/logging.py b/management_api_app/services/logging.py
new file mode 100644
--- /dev/null
+++ b/management_api_app/services/logging.py
@@ -0,0 +1,71 @@
+import logging
+import os
+
+from opencensus.ext.azure.log_exporter import AzureLogHandler
+from opencensus.trace import config_integration
+from opencensus.trace.samplers import AlwaysOnSampler
+from opencensus.trace.tracer import Tracer
+
+
+UNWANTED_LOGGERS = [
+    "azure.core.pipeline.policies.http_logging_policy",
+    "azure.eventhub._eventprocessor.event_processor",
+    "azure.identity.aio._credentials.managed_identity",
+    "azure.identity.aio._credentials.environment",
+    "azure.identity.aio._internal.get_token_mixin",
+    "azure.identity.aio._internal.decorators",
+    "azure.identity.aio._credentials.chained",
+    "azure.identity",
+    "msal.token_cache",
+    "uamqp",
+    "uamqp.authentication.cbs_auth_async",
+    "uamqp.async_ops.client_async",
+    "uamqp.async_ops.connection_async",
+    "uamqp.async_ops",
+    "uamqp.authentication",
+    "uamqp.c_uamqp",
+    "uamqp.connection",
+    "uamqp.receiver"
+]
+
+
+def disable_unwanted_loggers():
+    """
+    Disables the unwanted loggers.
+    """
+    for logger_name in UNWANTED_LOGGERS:
+        logging.getLogger(logger_name).disabled = True
+
+
+def initialize_logging(logging_level: int, correlation_id: str = None) -> logging.LoggerAdapter:
+    """
+    Adds the Application Insights handler for the root logger and sets the given logging level.
+    Creates and returns a logger adapter that integrates the correlation ID, if given, to the log messages.
+
+    :param logging_level: The logging level to set e.g., logging.WARNING.
+    :param correlation_id: Optional. The correlation ID that is passed on to the operation_Id in App Insights.
+    :returns: A newly created logger adapter.
+    """
+    logger = logging.getLogger()
+    logger.addHandler(logging.StreamHandler())  # For logging into console
+    app_insights_connection_string = os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING")
+
+    try:
+        logger.addHandler(AzureLogHandler(connection_string=app_insights_connection_string))
+    except ValueError as e:
+        logger.error(f"Failed to set Application Insights logger handler: {e}")
+
+    config_integration.trace_integrations(['logging'])
+    logging.basicConfig(level=logging_level, format='%(asctime)s traceId=%(traceId)s spanId=%(spanId)s %(message)s')
+    Tracer(sampler=AlwaysOnSampler())
+    logger.setLevel(logging_level)
+
+    extra = None
+
+    if correlation_id:
+        extra = {'traceId': correlation_id}
+
+    adapter = logging.LoggerAdapter(logger, extra)
+    adapter.debug(f"Logger adapter initialized with extra: {extra}")
+
+    return adapter


Test patch: