Repository: PrefectHQ/prefect

Issue:
Cloud logging does not survive pickling
Currently our cloud logs are sent to a Queue, which is then popped from in a background thread to be sent to cloud.  If a TaskRunner is shipped off to a new dask worker, this queue doesn't survive and that TaskRunner fails to log.  We should refactor to ensure that the log queues are freshly generated after unpickling of _all objects_ with loggers.


Discussion:

Resolving patch:
diff --git a/src/prefect/utilities/logging.py b/src/prefect/utilities/logging.py
--- a/src/prefect/utilities/logging.py
+++ b/src/prefect/utilities/logging.py
@@ -3,6 +3,7 @@
 import logging
 import os
 import queue
+import time
 from logging.handlers import QueueHandler, QueueListener
 
 import prefect
@@ -14,13 +15,19 @@ def __init__(self) -> None:
         super().__init__()
         self.logger_server = config.cloud.log
         self.client = None
+        self.errored_out = False
 
     def emit(self, record):
-        if self.client is None:
-            from prefect.client import Client
+        try:
+            if self.errored_out is True:
+                return
+            if self.client is None:
+                from prefect.client import Client
 
-            self.client = Client()
-        r = self.client.post(path="", server=self.logger_server, **record.__dict__)
+                self.client = Client()
+            r = self.client.post(path="", server=self.logger_server, **record.__dict__)
+        except:
+            self.errored_out = True
 
 
 old_factory = logging.getLogRecordFactory()
@@ -33,17 +40,24 @@ def cloud_record_factory(*args, **kwargs):
     return record
 
 
-def configure_logging() -> logging.Logger:
+def configure_logging(testing=False) -> logging.Logger:
     """
     Creates a "prefect" root logger with a `StreamHandler` that has level and formatting
     set from `prefect.config`.
 
+    Args:
+        - testing (bool, optional): a boolean specifying whether this configuration
+            is for testing purposes only; this helps us isolate any global state during testing
+            by configuring a "prefect-test-logger" instead of the standard "prefect" logger
+
     Returns:
         - logging.Logger
     """
-    logger = logging.getLogger("prefect")
+    name = "prefect-test-logger" if testing else "prefect"
+    logger = logging.getLogger(name)
     handler = logging.StreamHandler()
     formatter = logging.Formatter(config.logging.format)
+    formatter.converter = time.gmtime
     handler.setFormatter(formatter)
     logger.addHandler(handler)
     logger.setLevel(config.logging.level)
@@ -51,14 +65,8 @@ def configure_logging() -> logging.Logger:
     # send logs to server
     if config.logging.log_to_cloud:
         logging.setLogRecordFactory(cloud_record_factory)
-        log_queue = queue.Queue(-1)  # unlimited size queue
-        queue_handler = QueueHandler(log_queue)
         remote_handler = RemoteHandler()
-        remote_listener = QueueListener(log_queue, remote_handler)
-        logger.addHandler(queue_handler)
-        remote_listener.start()
-        stopper = lambda listener: listener.stop()
-        atexit.register(stopper, remote_listener)
+        logger.addHandler(remote_handler)
 
     return logger
 


Test patch:
diff --git a/tests/utilities/test_logging.py b/tests/utilities/test_logging.py
--- a/tests/utilities/test_logging.py
+++ b/tests/utilities/test_logging.py
@@ -7,37 +7,49 @@
 def test_root_logger_level_responds_to_config():
     try:
         with utilities.configuration.set_temporary_config({"logging.level": "DEBUG"}):
-            utilities.logging.configure_logging().level == logging.DEBUG
+            assert (
+                utilities.logging.configure_logging(testing=True).level == logging.DEBUG
+            )
 
         with utilities.configuration.set_temporary_config({"logging.level": "WARNING"}):
-            utilities.logging.configure_logging().level == logging.WARNING
+            assert (
+                utilities.logging.configure_logging(testing=True).level
+                == logging.WARNING
+            )
     finally:
         # reset root_logger
-        logger = utilities.logging.get_logger()
-        for h in logger.handlers:
-            logger.removeHandler(h)
-        utilities.logging.configure_logging()
+        logger = utilities.logging.configure_logging(testing=True)
+        logger.handlers = []
 
 
-def test_remote_handler_is_configured_for_cloud(monkeypatch):
-    starter = MagicMock()
-    listener = MagicMock(return_value=starter)
-    monkeypatch.setattr("prefect.utilities.logging.QueueListener", listener)
+def test_remote_handler_is_configured_for_cloud():
     try:
         with utilities.configuration.set_temporary_config(
             {"logging.log_to_cloud": True, "cloud.log": "http://foo.bar:1800/log"}
         ):
-            logger = utilities.logging.configure_logging()
-            assert listener.called
-            remote_handler = listener.call_args[0][1]
-            assert remote_handler.logger_server == "http://foo.bar:1800/log"
-            assert starter.start.called
+            logger = utilities.logging.configure_logging(testing=True)
+            assert hasattr(logger.handlers[-1], "client")
+            assert logger.handlers[-1].logger_server == "http://foo.bar:1800/log"
     finally:
         # reset root_logger
-        logger = utilities.logging.get_logger()
-        for h in logger.handlers:
-            logger.removeHandler(h)
-        utilities.logging.configure_logging()
+        logger = utilities.logging.configure_logging(testing=True)
+        logger.handlers = []
+
+
+def test_remote_handler_captures_errors_then_passes():
+    try:
+        with utilities.configuration.set_temporary_config(
+            {"logging.log_to_cloud": True, "cloud.log": "http://foo.bar:1800/log"}
+        ):
+            logger = utilities.logging.configure_logging(testing=True)
+            assert hasattr(logger.handlers[-1], "client")
+            child_logger = logger.getChild("sub-test")
+            child_logger.critical("this should raise an error in the handler")
+            assert logger.handlers[-1].errored_out == True
+    finally:
+        # reset root_logger
+        logger = utilities.logging.configure_logging(testing=True)
+        logger.handlers = []
 
 
 def test_get_logger_returns_root_logger():