Repository: PrefectHQ/prefect

Issue:
Add Agent Logging for Platform Roadblocks
## Use Case
e.g.
```
2019-09-11 20:06:34,237 - agent - INFO - Found 1 flow run(s) to submit for execution.
2019-09-11 20:06:34,545 - agent - ERROR - (403)
Reason: Forbidden
HTTP response headers: HTTPHeaderDict({'Audit-Id': '8775dbb3-ad97-43b6-9f0a-144b95f5b43b', 'Content-Type': 'application/json', 'X-Content-Type-Options': 'nosniff', 'Date': 'Wed, 11 Sep 2019 20:06:34 GMT', 'Content-Length': '309'})
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"jobs.batch is forbidden: User \"system:serviceaccount:my-space:default\" cannot create resource \"jobs\" in API group \"batch\" in the namespace \"my-space\"","reason":"Forbidden","details":{"group":"batch","kind":"jobs"},"code":403}
```

We want to notify users of potential platform issues. ^ in this one the agent didn't have permission to create jobs in its namespace (RBAC wasn't properly configured).


## Solution
While we can't force permissions we can elevate this was a warning to users through the flow run logs.




## Alternatives
Another option is to feed into the agent heartbeat issue #1373 where instead of becoming a flow run log it could be reflected in the agents dashboard. 


Discussion:

Resolving patch:
diff --git a/src/prefect/agent/agent.py b/src/prefect/agent/agent.py
--- a/src/prefect/agent/agent.py
+++ b/src/prefect/agent/agent.py
@@ -10,6 +10,7 @@
 from prefect.engine.state import Submitted
 from prefect.utilities.exceptions import AuthorizationError
 from prefect.utilities.graphql import with_args
+from prefect.utilities.graphql import GraphQLResult
 
 
 ascii_name = r"""
@@ -150,6 +151,7 @@ def agent_process(self, tenant_id: str) -> bool:
                 )
         except Exception as exc:
             self.logger.error(exc)
+            self._log_flow_run_exceptions(flow_runs, exc)  # type: ignore
 
         return bool(flow_runs)
 
@@ -275,6 +277,23 @@ def update_states(self, flow_runs: list) -> None:
                         ),
                     )
 
+    def _log_flow_run_exceptions(self, flow_runs: list, exc: Exception) -> None:
+        """
+        Log platform issues to Prefect Cloud, attached to each flow run which
+        could not start.
+
+        Args:
+            - flow_runs (list): A list of GraphQLResult flow run objects
+            - exc (Exception): A caught exception to log
+        """
+        for flow_run in flow_runs:
+            self.client.write_run_log(
+                flow_run_id=flow_run.id,  # type: ignore
+                name="agent",
+                message=str(exc),
+                level="ERROR",
+            )
+
     def deploy_flows(self, flow_runs: list) -> None:
         """
         Meant to be overridden by a platform specific deployment option


Test patch:
diff --git a/tests/agent/test_agent.py b/tests/agent/test_agent.py
--- a/tests/agent/test_agent.py
+++ b/tests/agent/test_agent.py
@@ -255,3 +255,51 @@ def test_agent_process_no_runs_found(monkeypatch, runner_token):
     # Assert it doesn't return everything but all functions are called properly
     agent = Agent()
     assert not agent.agent_process("id")
+
+
+def test_agent_logs_flow_run_exceptions(monkeypatch, runner_token):
+    gql_return = MagicMock(
+        return_value=MagicMock(data=MagicMock(writeRunLog=MagicMock(success=True)))
+    )
+    client = MagicMock()
+    client.return_value.write_run_log = gql_return
+    monkeypatch.setattr("prefect.agent.agent.Client", MagicMock(return_value=client))
+
+    agent = Agent()
+    agent._log_flow_run_exceptions(
+        flow_runs=[
+            GraphQLResult(
+                {
+                    "id": "id",
+                    "serialized_state": Scheduled().serialize(),
+                    "version": 1,
+                    "task_runs": [
+                        GraphQLResult(
+                            {
+                                "id": "id",
+                                "version": 1,
+                                "serialized_state": Scheduled().serialize(),
+                            }
+                        )
+                    ],
+                }
+            )
+        ],
+        exc=ValueError("Error Here"),
+    )
+
+    assert client.write_run_log.called
+    client.write_run_log.assert_called_with(
+        flow_run_id="id", level="ERROR", message="Error Here", name="agent"
+    )
+
+
+def test_agent_process_raises_exception_and_logs(monkeypatch, runner_token):
+    client = MagicMock()
+    client.return_value.graphql.side_effect = ValueError("Error")
+    monkeypatch.setattr("prefect.agent.agent.Client", client)
+
+    agent = Agent()
+    with pytest.raises(Exception):
+        agent.agent_process("id")
+        assert client.write_run_log.called