Repository: PrefectHQ/prefect

Issue:
Kubernetes manifest image and namespace shortcuts should be optional
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar request and didn't find it.
- [X] I searched the Prefect documentation for this feature.

### Prefect Version

2.x

### Describe the current behavior

Reported by one of our community members: https://prefect-community.slack.com/archives/CL09KU1K7/p1666199414970169

The `image` and `namespace` shortcuts on the `KubernetesJob` block are not optional, so you cannot pass them `None` to avoid injecting them.  Even worse, if you provide a full manifest file (like with `KubernetesJob.job_from_file`) the manifest's values will be overwritten by the Prefect defaults.

### Describe the proposed behavior

Instead, we should make the `image`/`namespace` attributes optional (with their current defaults) but all `None` to disable the shortcuts.

### Example Use

_No response_

### Additional context

_No response_


Discussion:

Resolving patch:
diff --git a/src/prefect/infrastructure/kubernetes.py b/src/prefect/infrastructure/kubernetes.py
--- a/src/prefect/infrastructure/kubernetes.py
+++ b/src/prefect/infrastructure/kubernetes.py
@@ -5,7 +5,7 @@
 
 import anyio.abc
 import yaml
-from pydantic import Field, validator
+from pydantic import Field, root_validator, validator
 from typing_extensions import Literal
 
 from prefect.blocks.kubernetes import KubernetesClusterConfig
@@ -75,12 +75,19 @@ class KubernetesJob(Infrastructure):
         default="kubernetes-job", description="The type of infrastructure."
     )
     # shortcuts for the most common user-serviceable settings
-    image: str = Field(
-        default_factory=get_prefect_image_name,
-        description="The tag of a Docker image to use for the job. Defaults to the Prefect image.",
+    image: Optional[str] = Field(
+        default=None,
+        description=(
+            "The tag of a Docker image to use for the job. Defaults to the Prefect "
+            "image unless an image is already present in a provided job manifest."
+        ),
     )
-    namespace: str = Field(
-        default="default", description="The Kubernetes namespace to use for this job."
+    namespace: Optional[str] = Field(
+        default=None,
+        description=(
+            "The Kubernetes namespace to use for this job. Defaults to 'default' "
+            "unless a namespace is already present in a provided job manifest."
+        ),
     )
     service_account_name: Optional[str] = Field(
         default=None, description="The Kubernetes service account to use for this job."
@@ -163,6 +170,33 @@ def cast_customizations_to_a_json_patch(
             return JsonPatch(value)
         return value
 
+    @root_validator
+    def default_namespace(cls, values):
+        job = values.get("job")
+
+        namespace = values.get("namespace")
+        job_namespace = job["metadata"].get("namespace") if job else None
+
+        if not namespace and not job_namespace:
+            values["namespace"] = "default"
+
+        return values
+
+    @root_validator
+    def default_image(cls, values):
+        job = values.get("job")
+        image = values.get("image")
+        job_image = (
+            job["spec"]["template"]["spec"]["containers"][0].get("image")
+            if job
+            else None
+        )
+
+        if not image and not job_image:
+            values["image"] = get_prefect_image_name()
+
+        return values
+
     # Support serialization of the 'JsonPatch' type
     class Config:
         arbitrary_types_allowed = True
@@ -278,18 +312,25 @@ def _shortcut_customizations(self) -> JsonPatch:
         """Produces the JSON 6902 patch for the most commonly used customizations, like
         image and namespace, which we offer as top-level parameters (with sensible
         default values)"""
-        shortcuts = [
-            {
-                "op": "add",
-                "path": "/metadata/namespace",
-                "value": self.namespace,
-            },
-            {
-                "op": "add",
-                "path": "/spec/template/spec/containers/0/image",
-                "value": self.image,
-            },
-        ]
+        shortcuts = []
+
+        if self.namespace:
+            shortcuts.append(
+                {
+                    "op": "add",
+                    "path": "/metadata/namespace",
+                    "value": self.namespace,
+                }
+            )
+
+        if self.image:
+            shortcuts.append(
+                {
+                    "op": "add",
+                    "path": "/spec/template/spec/containers/0/image",
+                    "value": self.image,
+                }
+            )
 
         shortcuts += [
             {


Test patch:
diff --git a/tests/blocks/checksums.json b/tests/blocks/checksums.json
--- a/tests/blocks/checksums.json
+++ b/tests/blocks/checksums.json
@@ -15,6 +15,6 @@
     "kubernetes-cluster-config": "sha256:90d421e948bfbe4cdc98b124995f0edd0f84b0837549ad1390423bad8e31cf3b",
     "docker-registry": "sha256:6db1457676eee0b54ca2834b06f80a80f7c90112e64f1bdd26afb2e62fcceed9",
     "docker-container": "sha256:ee3348b26a5d0e0226e2d3cf6f2e85a3592c3b19fb8a988dfccceee972b9b9eb",
-    "kubernetes-job": "sha256:686f931093d8fa3a80dee6eb66516be7b022bf29c44a38766da8571f25fede8b",
+    "kubernetes-job": "sha256:a27bd569f16c47d6e39f0c04d1724dc7a670ca5d6013d930c517b79954ee9b97",
     "process": "sha256:227b03037cf895d9a70603554b792f573e7e59b81a585d51a43ec1f826e81da3"
 }
diff --git a/tests/infrastructure/test_kubernetes_job.py b/tests/infrastructure/test_kubernetes_job.py
--- a/tests/infrastructure/test_kubernetes_job.py
+++ b/tests/infrastructure/test_kubernetes_job.py
@@ -205,6 +205,25 @@ def test_uses_image_setting(
     assert image == "foo"
 
 
+def test_allows_image_setting_from_manifest(
+    mock_k8s_client,
+    mock_watch,
+    mock_k8s_batch_client,
+):
+    mock_watch.stream = _mock_pods_stream_that_returns_running_pod
+
+    manifest = KubernetesJob.base_job_manifest()
+    manifest["spec"]["template"]["spec"]["containers"][0]["image"] = "test"
+    job = KubernetesJob(command=["echo", "hello"], job=manifest)
+    assert job.image is None
+    job.run(MagicMock())
+    mock_k8s_batch_client.create_namespaced_job.assert_called_once()
+    image = mock_k8s_batch_client.create_namespaced_job.call_args[0][1]["spec"][
+        "template"
+    ]["spec"]["containers"][0]["image"]
+    assert image == "test"
+
+
 def test_uses_labels_setting(
     mock_k8s_client,
     mock_watch,
@@ -355,6 +374,24 @@ def test_uses_namespace_setting(
     assert namespace == "foo"
 
 
+def test_allows_namespace_setting_from_manifest(
+    mock_k8s_client,
+    mock_watch,
+    mock_k8s_batch_client,
+):
+    mock_watch.stream = _mock_pods_stream_that_returns_running_pod
+    manifest = KubernetesJob.base_job_manifest()
+    manifest["metadata"]["namespace"] = "test"
+    job = KubernetesJob(command=["echo", "hello"], job=manifest)
+    assert job.namespace is None
+    job.run(MagicMock())
+    mock_k8s_batch_client.create_namespaced_job.assert_called_once()
+    namespace = mock_k8s_batch_client.create_namespaced_job.call_args[0][1]["metadata"][
+        "namespace"
+    ]
+    assert namespace == "test"
+
+
 def test_uses_service_account_name_setting(
     mock_k8s_client,
     mock_watch,