Repository: PrefectHQ/prefect

Issue:
Attempt to log possible log writing errors
## Current behavior
Currently if the logger encounters an error it isn't immediately surfaced to Cloud users what the reason is for the logging error.



## Proposed behavior
Something along the lines of (but not limited to):
```python
try:
   write_some_logs()
except Exception as exc:
    sleep(1)
    write_this_error_log(exc)
except Exception as exc2:
    raise exc    # raise exception as it normally would
```

Where if the writing of the logs fails then the exception is passed as an attempted log. Otherwise if the exception logging fails then raise as it normally would. This would give some nice insight into "missing" logs.

## Example
Possible issues such as non serializeable bytes, large payloads, etc. could be surfaced

```
self.logger.info(b"bad bytes string")
CRITICAL - CloudHandler | Failed to write log with error: Object of type bytes is not JSON serializable
```

```
self.logger.info("LARGE STRING...........")
CRITICAL - CloudHandler | Failed to write log with error: 413 Client Error: Request Entity Too Large for url: https://api.prefect.io/graphql/alpha
```


Discussion:

Resolving patch:
diff --git a/src/prefect/utilities/logging.py b/src/prefect/utilities/logging.py
--- a/src/prefect/utilities/logging.py
+++ b/src/prefect/utilities/logging.py
@@ -63,9 +63,15 @@ def batch_upload(self) -> None:
                 assert self.client is not None
                 self.client.write_run_logs(logs)
             except Exception as exc:
-                self.logger.critical(
-                    "Failed to write log with error: {}".format(str(exc))
-                )
+                message = "Failed to write log with error: {}".format(str(exc))
+                self.logger.critical(message)
+
+                # Attempt to write batch error log otherwise log invalid cloud communication
+                try:
+                    assert self.client is not None
+                    self.client.write_run_logs([self._make_error_log(message)])
+                except Exception as exc:
+                    self.logger.critical("Unable to write logs to Prefect Cloud")
 
     def _monitor(self) -> None:
         while not self._flush:
@@ -92,7 +98,10 @@ def put(self, log: dict) -> None:
             json.dumps(log)  # make sure the payload is serializable
             self.queue.put(log)
         except TypeError as exc:
-            self.logger.critical("Failed to write log with error: {}".format(str(exc)))
+            message = "Failed to write log with error: {}".format(str(exc))
+            self.logger.critical(message)
+
+            self.queue.put(self._make_error_log(message))
 
     def emit(self, record) -> None:  # type: ignore
         # if we shouldn't log to cloud, don't emit
@@ -125,7 +134,21 @@ def emit(self, record) -> None:  # type: ignore
             log["info"] = record_dict
             self.put(log)
         except Exception as exc:
-            self.logger.critical("Failed to write log with error: {}".format(str(exc)))
+            message = "Failed to write log with error: {}".format(str(exc))
+            self.logger.critical(message)
+
+            self.put(self._make_error_log(message))
+
+    def _make_error_log(self, message: str) -> dict:
+        log = dict()
+        log["flowRunId"] = prefect.context.get("flow_run_id", None)
+        log["timestamp"] = pendulum.from_timestamp(time.time()).isoformat()
+        log["name"] = self.logger.name
+        log["message"] = message
+        log["level"] = "CRITICAL"
+        log["info"] = {}
+
+        return log
 
 
 def configure_logging(testing: bool = False) -> logging.Logger:


Test patch:
diff --git a/tests/utilities/test_logging.py b/tests/utilities/test_logging.py
--- a/tests/utilities/test_logging.py
+++ b/tests/utilities/test_logging.py
@@ -3,7 +3,9 @@
 import time
 from unittest.mock import MagicMock
 
-from prefect import utilities
+import pytest
+
+from prefect import context, utilities
 
 
 def test_root_logger_level_responds_to_config():
@@ -169,7 +171,7 @@ def write_run_logs(self, *args, **kwargs):
         logger.handlers = []
 
 
-def test_cloud_handler_removes_bad_logs_from_queue_and_continues(caplog, monkeypatch):
+def test_cloud_handler_removes_bad_logs_from_queue_and_logs_error(caplog, monkeypatch):
     calls = []
 
     class Client:
@@ -192,13 +194,49 @@ def write_run_logs(self, *args, **kwargs):
         time.sleep(0.75)
         assert len(calls) == 1
         msgs = [c["message"] for c in calls[0]["args"][0]]
-        assert msgs == ["one", "three"]
+
+        assert msgs[0] == "one"
+        assert "is not JSON serializable" in msgs[1]
+        assert msgs[2] == "three"
+    finally:
+        # reset root_logger
+        logger = utilities.logging.configure_logging(testing=True)
+        logger.handlers = []
+
+
+def test_cloud_handler_client_error(caplog, monkeypatch):
+    class Client:
+        def write_run_logs(self, *args, **kwargs):
+            raise utilities.exceptions.ClientError("Error")
+
+    monkeypatch.setattr("prefect.client.Client", Client)
+    try:
+        logger = utilities.logging.configure_logging(testing=True)
+        cloud_handler = logger.handlers[-1]
+        assert isinstance(cloud_handler, utilities.logging.CloudHandler)
+
+        with utilities.configuration.set_temporary_config(
+            {"logging.log_to_cloud": True}
+        ):
+            logger.critical("one")
     finally:
         # reset root_logger
         logger = utilities.logging.configure_logging(testing=True)
         logger.handlers = []
 
 
+def test_make_error_log(caplog):
+
+    with context({"flow_run_id": "f_id", "task_run_id": "t_id"}):
+        log = utilities.logging.CloudHandler()._make_error_log("test_message")
+        assert log["flowRunId"] == "f_id"
+        assert log["timestamp"]
+        assert log["name"] == "CloudHandler"
+        assert log["message"] == "test_message"
+        assert log["level"] == "CRITICAL"
+        assert log["info"] == {}
+
+
 def test_get_logger_returns_root_logger():
     assert utilities.logging.get_logger() is logging.getLogger("prefect")