Repository: PrefectHQ/prefect

Issue:
Store and Display Worker Logs in Prefect UI
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar request and didn't find it.
- [X] I searched the Prefect documentation for this feature.

### Prefect Version

2.x

### Describe the current behavior

Worker logs are present in execution environment logs, but not stored / displayed in the UI.

### Describe the proposed behavior

As a first step, we should store and display worker logs associated with a flow run and display them interleaved with execution logs as one would see in their terminal. Later on we can consider worker logs that are not associated with flow runs, such as polling logs, to display on (future) worker pages.

### Example Use

_No response_

### Additional context

_No response_


Discussion:
Per this discussion, if we could print the name of the worker that would be great

https://github.com/PrefectHQ/prefect/pull/7362

Resolving patch:
diff --git a/src/prefect/logging/loggers.py b/src/prefect/logging/loggers.py
--- a/src/prefect/logging/loggers.py
+++ b/src/prefect/logging/loggers.py
@@ -4,12 +4,13 @@
 from builtins import print
 from contextlib import contextmanager
 from functools import lru_cache
-from typing import TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, Dict, Optional, Union
 
 import prefect
 from prefect.exceptions import MissingContextError
 
 if TYPE_CHECKING:
+    from prefect.client.schemas import FlowRun as ClientFlowRun
     from prefect.context import RunContext
     from prefect.flows import Flow
     from prefect.server.schemas.core import FlowRun, TaskRun
@@ -30,6 +31,20 @@ def process(self, msg, kwargs):
         kwargs["extra"] = {**self.extra, **(kwargs.get("extra") or {})}
         return (msg, kwargs)
 
+    def getChild(
+        self, suffix: str, extra: Optional[Dict[str, str]] = None
+    ) -> "PrefectLogAdapter":
+        if extra is None:
+            extra = {}
+
+        return PrefectLogAdapter(
+            self.logger.getChild(suffix),
+            extra={
+                **self.extra,
+                **extra,
+            },
+        )
+
 
 @lru_cache()
 def get_logger(name: str = None) -> logging.Logger:
@@ -117,7 +132,11 @@ def get_run_logger(
     return logger
 
 
-def flow_run_logger(flow_run: "FlowRun", flow: "Flow" = None, **kwargs: str):
+def flow_run_logger(
+    flow_run: Union["FlowRun", "ClientFlowRun"],
+    flow: Optional["Flow"] = None,
+    **kwargs: str,
+):
     """
     Create a flow run logger with the run's metadata attached.
 
diff --git a/src/prefect/workers/base.py b/src/prefect/workers/base.py
--- a/src/prefect/workers/base.py
+++ b/src/prefect/workers/base.py
@@ -22,7 +22,7 @@
     InfrastructureNotFound,
     ObjectNotFound,
 )
-from prefect.logging.loggers import get_logger
+from prefect.logging.loggers import PrefectLogAdapter, get_logger, flow_run_logger
 from prefect.server import schemas
 from prefect.server.schemas.actions import WorkPoolUpdate
 from prefect.server.schemas.filters import (
@@ -390,6 +390,18 @@ def get_default_base_job_template(cls) -> Dict:
     def get_name_slug(self):
         return slugify(self.name)
 
+    def get_flow_run_logger(self, flow_run: "FlowRun") -> PrefectLogAdapter:
+        return flow_run_logger(flow_run=flow_run).getChild(
+            "worker",
+            extra={
+                "worker_name": self.name,
+                "work_pool_name": (
+                    self._work_pool_name if self._work_pool else "<unknown>"
+                ),
+                "work_pool_id": str(getattr(self._work_pool, "id", "unknown")),
+            },
+        )
+
     @abc.abstractmethod
     async def run(
         self,
@@ -512,8 +524,10 @@ async def check_for_cancelled_flow_runs(self):
         return cancelling_flow_runs
 
     async def cancel_run(self, flow_run: "FlowRun"):
+        run_logger = self.get_flow_run_logger(flow_run)
+
         if not flow_run.infrastructure_pid:
-            self._logger.error(
+            run_logger.error(
                 f"Flow run '{flow_run.id}' does not have an infrastructure pid"
                 " attached. Cancellation cannot be guaranteed."
             )
@@ -546,7 +560,7 @@ async def cancel_run(self, flow_run: "FlowRun"):
         except InfrastructureNotAvailable as exc:
             self._logger.warning(f"{exc} Flow run cannot be cancelled by this worker.")
         except Exception:
-            self._logger.exception(
+            run_logger.exception(
                 "Encountered exception while killing infrastructure for flow run "
                 f"'{flow_run.id}'. Flow run may not be cancelled."
             )
@@ -558,7 +572,7 @@ async def cancel_run(self, flow_run: "FlowRun"):
                 flow_run=flow_run, configuration=configuration
             )
             await self._mark_flow_run_as_cancelled(flow_run)
-            self._logger.info(f"Cancelled flow run '{flow_run.id}'!")
+            run_logger.info(f"Cancelled flow run '{flow_run.id}'!")
 
     async def _update_local_work_pool_info(self):
         try:
@@ -663,7 +677,10 @@ async def _submit_scheduled_flow_runs(
                 )
                 break
             else:
-                self._logger.info(f"Submitting flow run '{flow_run.id}'")
+                run_logger = self.get_flow_run_logger(flow_run)
+                run_logger.info(
+                    f"Worker '{self.name}' submitting flow run '{flow_run.id}'"
+                )
                 self._submitting_flow_run_ids.add(flow_run.id)
                 self._runs_task_group.start_soon(
                     self._submit_run,
@@ -696,6 +713,8 @@ async def _submit_run(self, flow_run: "FlowRun") -> None:
         """
         Submits a given flow run for execution by the worker.
         """
+        run_logger = self.get_flow_run_logger(flow_run)
+
         try:
             await self._check_flow_run(flow_run)
         except ValueError:
@@ -723,13 +742,13 @@ async def _submit_run(self, flow_run: "FlowRun") -> None:
                         infrastructure_pid=str(readiness_result),
                     )
                 except Exception:
-                    self._logger.exception(
+                    run_logger.exception(
                         "An error occurred while setting the `infrastructure_pid` on "
                         f"flow run {flow_run.id!r}. The flow run will "
                         "not be cancellable."
                     )
 
-            self._logger.info(f"Completed submission of flow run '{flow_run.id}'")
+            run_logger.info(f"Completed submission of flow run '{flow_run.id}'")
 
         else:
             # If the run is not ready to submit, release the concurrency slot
@@ -741,6 +760,8 @@ async def _submit_run(self, flow_run: "FlowRun") -> None:
     async def _submit_run_and_capture_errors(
         self, flow_run: "FlowRun", task_status: anyio.abc.TaskStatus = None
     ) -> Union[BaseWorkerResult, Exception]:
+        run_logger = self.get_flow_run_logger(flow_run)
+
         try:
             configuration = await self._get_configuration(flow_run)
             submitted_event = self._emit_flow_run_submitted_event(configuration)
@@ -759,7 +780,7 @@ async def _submit_run_and_capture_errors(
                 task_status.started(exc)
                 await self._propose_failed_state(flow_run, exc)
             else:
-                self._logger.exception(
+                run_logger.exception(
                     f"An error occurred while monitoring flow run '{flow_run.id}'. "
                     "The flow run will not be marked as failed, but an issue may have "
                     "occurred."
@@ -770,7 +791,7 @@ async def _submit_run_and_capture_errors(
                 self._limiter.release_on_behalf_of(flow_run.id)
 
         if not task_status._future.done():
-            self._logger.error(
+            run_logger.error(
                 f"Infrastructure returned without reporting flow run '{flow_run.id}' "
                 "as started or raising an error. This behavior is not expected and "
                 "generally indicates improper implementation of infrastructure. The "
@@ -826,13 +847,14 @@ async def _get_configuration(
         return configuration
 
     async def _propose_pending_state(self, flow_run: "FlowRun") -> bool:
+        run_logger = self.get_flow_run_logger(flow_run)
         state = flow_run.state
         try:
             state = await propose_state(
                 self._client, Pending(), flow_run_id=flow_run.id
             )
         except Abort as exc:
-            self._logger.info(
+            run_logger.info(
                 (
                     f"Aborted submission of flow run '{flow_run.id}'. "
                     f"Server sent an abort signal: {exc}"
@@ -840,13 +862,13 @@ async def _propose_pending_state(self, flow_run: "FlowRun") -> bool:
             )
             return False
         except Exception:
-            self._logger.exception(
+            run_logger.exception(
                 f"Failed to update state of flow run '{flow_run.id}'",
             )
             return False
 
         if not state.is_pending():
-            self._logger.info(
+            run_logger.info(
                 (
                     f"Aborted submission of flow run '{flow_run.id}': "
                     f"Server returned a non-pending state {state.type.value!r}"
@@ -857,6 +879,7 @@ async def _propose_pending_state(self, flow_run: "FlowRun") -> bool:
         return True
 
     async def _propose_failed_state(self, flow_run: "FlowRun", exc: Exception) -> None:
+        run_logger = self.get_flow_run_logger(flow_run)
         try:
             await propose_state(
                 self._client,
@@ -868,12 +891,13 @@ async def _propose_failed_state(self, flow_run: "FlowRun", exc: Exception) -> No
             # raise in the agent process
             pass
         except Exception:
-            self._logger.error(
+            run_logger.error(
                 f"Failed to update state of flow run '{flow_run.id}'",
                 exc_info=True,
             )
 
     async def _propose_crashed_state(self, flow_run: "FlowRun", message: str) -> None:
+        run_logger = self.get_flow_run_logger(flow_run)
         try:
             state = await propose_state(
                 self._client,
@@ -884,12 +908,10 @@ async def _propose_crashed_state(self, flow_run: "FlowRun", message: str) -> Non
             # Flow run already marked as failed
             pass
         except Exception:
-            self._logger.exception(
-                f"Failed to update state of flow run '{flow_run.id}'"
-            )
+            run_logger.exception(f"Failed to update state of flow run '{flow_run.id}'")
         else:
             if state.is_crashed():
-                self._logger.info(
+                run_logger.info(
                     f"Reported flow run '{flow_run.id}' as crashed: {message}"
                 )
 
diff --git a/src/prefect/workers/process.py b/src/prefect/workers/process.py
--- a/src/prefect/workers/process.py
+++ b/src/prefect/workers/process.py
@@ -145,6 +145,8 @@ async def run(
         if not command:
             command = f"{sys.executable} -m prefect.engine"
 
+        flow_run_logger = self.get_flow_run_logger(flow_run)
+
         # We must add creationflags to a dict so it is only passed as a function
         # parameter on Windows, because the presence of creationflags causes
         # errors on Unix even if set to None
@@ -153,7 +155,7 @@ async def run(
             kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
 
         _use_threaded_child_watcher()
-        self._logger.info("Opening process...")
+        flow_run_logger.info("Opening process...")
 
         working_dir_ctx = (
             tempfile.TemporaryDirectory(suffix="prefect")
@@ -161,7 +163,9 @@ async def run(
             else contextlib.nullcontext(configuration.working_dir)
         )
         with working_dir_ctx as working_dir:
-            self._logger.debug(f"Process running command: {command} in {working_dir}")
+            flow_run_logger.debug(
+                f"Process running command: {command} in {working_dir}"
+            )
             process = await run_process(
                 command.split(" "),
                 stream_output=configuration.stream_output,
@@ -202,12 +206,12 @@ async def run(
                     "Typically, this is caused by manual cancellation."
                 )
 
-            self._logger.error(
+            flow_run_logger.error(
                 f"Process{display_name} exited with status code: {process.returncode}"
                 + (f"; {help_message}" if help_message else "")
             )
         else:
-            self._logger.info(f"Process{display_name} exited cleanly.")
+            flow_run_logger.info(f"Process{display_name} exited cleanly.")
 
         return ProcessWorkerResult(
             status_code=process.returncode, identifier=str(process.pid)


Test patch:
diff --git a/tests/test_logging.py b/tests/test_logging.py
--- a/tests/test_logging.py
+++ b/tests/test_logging.py
@@ -33,6 +33,7 @@
 from prefect.logging.handlers import APILogHandler, APILogWorker, PrefectConsoleHandler
 from prefect.logging.highlighters import PrefectConsoleHighlighter
 from prefect.logging.loggers import (
+    PrefectLogAdapter,
     disable_logger,
     disable_run_logger,
     flow_run_logger,
@@ -1361,3 +1362,12 @@ def my_flow():
     assert record.name == "prefect.flow_runs"
     assert record.flow_run_id == str(flow_run.id)
     assert record.flow_name == my_flow.name
+
+
+def test_log_adapter_get_child(flow_run):
+    logger = PrefectLogAdapter(get_logger("prefect.parent"), {"hello": "world"})
+    assert logger.extra == {"hello": "world"}
+
+    child_logger = logger.getChild("child", {"goodnight": "moon"})
+    assert child_logger.logger.name == "prefect.parent.child"
+    assert child_logger.extra == {"hello": "world", "goodnight": "moon"}
diff --git a/tests/workers/test_base_worker.py b/tests/workers/test_base_worker.py
--- a/tests/workers/test_base_worker.py
+++ b/tests/workers/test_base_worker.py
@@ -1695,3 +1695,27 @@ async def run(self, flow_run, configuration, task_status=None):
             in caplog.text
         )
         assert "Cancellation cannot be guaranteed." in caplog.text
+
+
+async def test_get_flow_run_logger(
+    orion_client: PrefectClient, worker_deployment_wq1, work_pool
+):
+    flow_run = await orion_client.create_flow_run_from_deployment(
+        worker_deployment_wq1.id
+    )
+
+    async with WorkerTestImpl(
+        name="test", work_pool_name="test-work-pool", create_pool_if_not_found=False
+    ) as worker:
+        await worker.sync_with_backend()
+        logger = worker.get_flow_run_logger(flow_run)
+
+        assert logger.name == "prefect.flow_runs.worker"
+        assert logger.extra == {
+            "flow_run_name": flow_run.name,
+            "flow_run_id": str(flow_run.id),
+            "flow_name": "<unknown>",
+            "worker_name": "test",
+            "work_pool_name": "test-work-pool",
+            "work_pool_id": str(work_pool.id),
+        }