Repository: PrefectHQ/prefect

Issue:
Make attachment of kubernetes logger to DaskKubernetes env opt-in
## Current behavior
<!-- Please describe how the feature works today -->
The DaskKubernetes environment automatically attaches Kubernetes loggers to Prefect Cloud logs.  This is a great feature for observability, but will inevitably wind up accidentally shipping very sensitive information from users to Prefect Cloud logs unintentionally.



## Proposed behavior
<!-- Please describe your proposed change to the current behavior -->
Perhaps there is a middle ground where, rather than having to explicitly declare the kubernetes logger, a config flag can be set to add kubernetes loggers, but it should default to False.  There may be some additional notes to add to the docs around this so that users know to turn this on to help with debugging.



## Example
<!-- Please give an example of how the enhancement would be useful -->



Discussion:

Resolving patch:
diff --git a/src/prefect/environments/execution/dask/k8s.py b/src/prefect/environments/execution/dask/k8s.py
--- a/src/prefect/environments/execution/dask/k8s.py
+++ b/src/prefect/environments/execution/dask/k8s.py
@@ -75,6 +75,8 @@ class DaskKubernetesEnvironment(Environment):
         - image_pull_secret (str, optional): optional name of an `imagePullSecret` to use for
             the scheduler and worker pods. For more information go
             [here](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/).
+        - log_k8s_errors (bool, optional): optional toggle to also log Kubernetes errors that may occur
+            using the Prefect logger. Defaults to `False`.
     """
 
     def __init__(
@@ -92,6 +94,7 @@ def __init__(
         scheduler_spec_file: str = None,
         worker_spec_file: str = None,
         image_pull_secret: str = None,
+        log_k8s_errors: bool = False,
     ) -> None:
         self.min_workers = min_workers
         self.max_workers = max_workers
@@ -116,6 +119,7 @@ def __init__(
         self._scheduler_spec, self._worker_spec = self._load_specs_from_file()
 
         self._identifier_label = ""
+        self.log_k8s_errors = log_k8s_errors
 
         super().__init__(
             labels=labels, on_start=on_start, on_exit=on_exit, metadata=metadata
@@ -322,8 +326,11 @@ def _extra_loggers(self) -> str:
         cluster_loggers = [
             "dask_kubernetes.core",
             "distributed.deploy.adaptive",
-            "kubernetes",
         ]
+
+        if self.log_k8s_errors:
+            cluster_loggers.append("kubernetes")
+
         config_extra_loggers = prefect.config.logging.extra_loggers
 
         extra_loggers = [*config_extra_loggers, *cluster_loggers]


Test patch:
diff --git a/tests/environments/execution/test_dask_k8s_environment.py b/tests/environments/execution/test_dask_k8s_environment.py
--- a/tests/environments/execution/test_dask_k8s_environment.py
+++ b/tests/environments/execution/test_dask_k8s_environment.py
@@ -232,7 +232,9 @@ def test_environment_run(monkeypatch):
 
 
 def test_populate_job_yaml():
-    environment = DaskKubernetesEnvironment(work_stealing=True, scheduler_logs=True)
+    environment = DaskKubernetesEnvironment(
+        work_stealing=True, scheduler_logs=True, log_k8s_errors=True
+    )
 
     file_path = os.path.dirname(prefect.environments.execution.dask.k8s.__file__)
 
@@ -314,7 +316,7 @@ def test_populate_worker_pod_yaml():
     assert env[2]["value"] == "id_test"
     assert (
         env[10]["value"]
-        == "['test_logger', 'dask_kubernetes.core', 'distributed.deploy.adaptive', 'kubernetes']"
+        == "['test_logger', 'dask_kubernetes.core', 'distributed.deploy.adaptive']"
     )
 
     assert yaml_obj["spec"]["containers"][0]["image"] == "my_image"
@@ -415,7 +417,7 @@ def test_populate_custom_worker_spec_yaml(log_flag):
     assert env[7]["value"] == str(log_flag).lower()
     assert (
         env[8]["value"]
-        == "['test_logger', 'dask_kubernetes.core', 'distributed.deploy.adaptive', 'kubernetes']"
+        == "['test_logger', 'dask_kubernetes.core', 'distributed.deploy.adaptive']"
     )
 
     assert yaml_obj["spec"]["containers"][0]["image"] == "my_image"
@@ -462,7 +464,7 @@ def test_populate_custom_scheduler_spec_yaml(log_flag):
     assert env[9]["value"] == str(log_flag).lower()
     assert (
         env[10]["value"]
-        == "['test_logger', 'dask_kubernetes.core', 'distributed.deploy.adaptive', 'kubernetes']"
+        == "['test_logger', 'dask_kubernetes.core', 'distributed.deploy.adaptive']"
     )
 
     assert (