Repository: kubeflow/pipelines

Issue:
[sdk]  Missing optional parameter to  v2 `use_config_map_as_volume` and `use_secret_as_volume` 
In kfp v1 the config map or secret can be marked as optional when used as a volume source, as shown in the example below.

This capability is missing in the recent added `use_config_map_as_volume` and `use_secret_as_volume` v2 sdk functions. (https://github.com/kubeflow/pipelines/pull/10400, https://github.com/kubeflow/pipelines/pull/10483)

```bash
vol = k8s_client.V1Volume(
        name='dummy',
        config_map=k8s_client.V1ConfigMapVolumeSource(name='name', optional=True),
    )
    component.add_pvolumes({mountPoint: vol})


vol = k8s_client.V1Volume(
        name='dummy',
        secret=k8s_client.V1SecretVolumeSource(secret_name='name', optional=False),
    )
    component.add_pvolumes({mountPoint: vol})

```


What is the use case or pain point?

Making the secret or config map an optional resource allows the KFP component to function without failing even if those resources are not present.




Discussion:

Resolving patch:
diff --git a/kubernetes_platform/python/kfp/kubernetes/config_map.py b/kubernetes_platform/python/kfp/kubernetes/config_map.py
--- a/kubernetes_platform/python/kfp/kubernetes/config_map.py
+++ b/kubernetes_platform/python/kfp/kubernetes/config_map.py
@@ -61,6 +61,7 @@ def use_config_map_as_volume(
     task: PipelineTask,
     config_map_name: str,
     mount_path: str,
+    optional: bool = False,
 ) -> PipelineTask:
     """Use a Kubernetes ConfigMap by mounting its data to the task's container as
     described by the `Kubernetes documentation <https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#add-configmap-data-to-a-volume>`_.
@@ -69,6 +70,7 @@ def use_config_map_as_volume(
         task: Pipeline task.
         config_map_name: Name of the ConfigMap.
         mount_path: Path to which to mount the ConfigMap data.
+        optional: Optional field specifying whether the ConfigMap must be defined.
 
     Returns:
         Task object with updated ConfigMap configuration.
@@ -79,6 +81,7 @@ def use_config_map_as_volume(
     config_map_as_vol = pb.ConfigMapAsVolume(
         config_map_name=config_map_name,
         mount_path=mount_path,
+        optional=optional,
     )
     msg.config_map_as_volume.append(config_map_as_vol)
 
diff --git a/kubernetes_platform/python/kfp/kubernetes/secret.py b/kubernetes_platform/python/kfp/kubernetes/secret.py
--- a/kubernetes_platform/python/kfp/kubernetes/secret.py
+++ b/kubernetes_platform/python/kfp/kubernetes/secret.py
@@ -61,6 +61,7 @@ def use_secret_as_volume(
     task: PipelineTask,
     secret_name: str,
     mount_path: str,
+    optional: bool = False,
 ) -> PipelineTask:
     """Use a Kubernetes Secret by mounting its data to the task's container as
     described by the `Kubernetes documentation <https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-files-from-a-pod>`_.
@@ -69,6 +70,7 @@ def use_secret_as_volume(
         task: Pipeline task.
         secret_name: Name of the Secret.
         mount_path: Path to which to mount the Secret data.
+        optional: Optional field specifying whether the Secret must be defined.
 
     Returns:
         Task object with updated secret configuration.
@@ -79,6 +81,7 @@ def use_secret_as_volume(
     secret_as_vol = pb.SecretAsVolume(
         secret_name=secret_name,
         mount_path=mount_path,
+        optional=optional,
     )
 
     msg.secret_as_volume.append(secret_as_vol)


Test patch:
diff --git a/backend/src/v2/driver/driver_test.go b/backend/src/v2/driver/driver_test.go
--- a/backend/src/v2/driver/driver_test.go
+++ b/backend/src/v2/driver/driver_test.go
@@ -530,7 +530,87 @@ func Test_extendPodSpecPatch_Secret(t *testing.T) {
 					{
 						Name: "secret1",
 						VolumeSource: k8score.VolumeSource{
-							Secret: &k8score.SecretVolumeSource{SecretName: "secret1"},
+							Secret: &k8score.SecretVolumeSource{SecretName: "secret1", Optional:  &[]bool{false}[0],},
+						},
+					},
+				},
+			},
+		},
+		{
+			"Valid - secret as volume with optional false",
+			&kubernetesplatform.KubernetesExecutorConfig{
+				SecretAsVolume: []*kubernetesplatform.SecretAsVolume{
+					{
+						SecretName: "secret1",
+						MountPath:  "/data/path",
+						Optional:   &[]bool{false}[0],
+					},
+				},
+			},
+			&k8score.PodSpec{
+				Containers: []k8score.Container{
+					{
+						Name: "main",
+					},
+				},
+			},
+			&k8score.PodSpec{
+				Containers: []k8score.Container{
+					{
+						Name: "main",
+						VolumeMounts: []k8score.VolumeMount{
+							{
+								Name:      "secret1",
+								MountPath: "/data/path",
+							},
+						},
+					},
+				},
+				Volumes: []k8score.Volume{
+					{
+						Name: "secret1",
+						VolumeSource: k8score.VolumeSource{
+							Secret: &k8score.SecretVolumeSource{SecretName: "secret1", Optional: &[]bool{false}[0]},
+						},
+					},
+				},
+			},
+		},
+		{
+			"Valid - secret as volume with optional true",
+			&kubernetesplatform.KubernetesExecutorConfig{
+				SecretAsVolume: []*kubernetesplatform.SecretAsVolume{
+					{
+						SecretName: "secret1",
+						MountPath:  "/data/path",
+						Optional:   &[]bool{true}[0],
+					},
+				},
+			},
+			&k8score.PodSpec{
+				Containers: []k8score.Container{
+					{
+						Name: "main",
+					},
+				},
+			},
+			&k8score.PodSpec{
+				Containers: []k8score.Container{
+					{
+						Name: "main",
+						VolumeMounts: []k8score.VolumeMount{
+							{
+								Name:      "secret1",
+								MountPath: "/data/path",
+							},
+						},
+					},
+				},
+				Volumes: []k8score.Volume{
+					{
+						Name: "secret1",
+						VolumeSource: k8score.VolumeSource{
+							Secret: &k8score.SecretVolumeSource{SecretName: "secret1", Optional: &[]bool{true}[0]},
 						},
 					},
 				},
@@ -647,7 +727,92 @@ func Test_extendPodSpecPatch_ConfigMap(t *testing.T) {
 						Name: "cm1",
 						VolumeSource: k8score.VolumeSource{
 							ConfigMap: &k8score.ConfigMapVolumeSource{
-								LocalObjectReference: k8score.LocalObjectReference{Name: "cm1"}},
+								LocalObjectReference: k8score.LocalObjectReference{Name: "cm1"},
+								Optional:             &[]bool{false}[0],},
+						},
+					},
+				},
+			},
+		},
+		{
+			"Valid - config map as volume with optional false",
+			&kubernetesplatform.KubernetesExecutorConfig{
+				ConfigMapAsVolume: []*kubernetesplatform.ConfigMapAsVolume{
+					{
+						ConfigMapName: "cm1",
+						MountPath:     "/data/path",
+						Optional:      &[]bool{false}[0],
+					},
+				},
+			},
+			&k8score.PodSpec{
+				Containers: []k8score.Container{
+					{
+						Name: "main",
+					},
+				},
+			},
+			&k8score.PodSpec{
+				Containers: []k8score.Container{
+					{
+						Name: "main",
+						VolumeMounts: []k8score.VolumeMount{
+							{
+								Name:      "cm1",
+								MountPath: "/data/path",
+							},
+						},
+					},
+				},
+				Volumes: []k8score.Volume{
+					{
+						Name: "cm1",
+						VolumeSource: k8score.VolumeSource{
+							ConfigMap: &k8score.ConfigMapVolumeSource{
+								LocalObjectReference: k8score.LocalObjectReference{Name: "cm1"},
+								Optional:             &[]bool{false}[0]},
+						},
+					},
+				},
+			},
+		},
+		{
+			"Valid - config map as volume with optional true",
+			&kubernetesplatform.KubernetesExecutorConfig{
+				ConfigMapAsVolume: []*kubernetesplatform.ConfigMapAsVolume{
+					{
+						ConfigMapName: "cm1",
+						MountPath:     "/data/path",
+						Optional:      &[]bool{true}[0],
+					},
+				},
+			},
+			&k8score.PodSpec{
+				Containers: []k8score.Container{
+					{
+						Name: "main",
+					},
+				},
+			},
+			&k8score.PodSpec{
+				Containers: []k8score.Container{
+					{
+						Name: "main",
+						VolumeMounts: []k8score.VolumeMount{
+							{
+								Name:      "cm1",
+								MountPath: "/data/path",
+							},
+						},
+					},
+				},
+				Volumes: []k8score.Volume{
+					{
+						Name: "cm1",
+						VolumeSource: k8score.VolumeSource{
+							ConfigMap: &k8score.ConfigMapVolumeSource{
+								LocalObjectReference: k8score.LocalObjectReference{Name: "cm1"},
+								Optional:             &[]bool{true}[0]},
 						},
 					},
 				},
diff --git a/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml b/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml
--- a/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml
+++ b/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml
@@ -56,3 +56,4 @@ platforms:
           configMapAsVolume:
           - mountPath: /mnt/my_vol
             configMapName: my-cm
+            optional: False
diff --git a/kubernetes_platform/python/test/snapshot/data/secret_as_vol.yaml b/kubernetes_platform/python/test/snapshot/data/secret_as_vol.yaml
--- a/kubernetes_platform/python/test/snapshot/data/secret_as_vol.yaml
+++ b/kubernetes_platform/python/test/snapshot/data/secret_as_vol.yaml
@@ -56,3 +56,4 @@ platforms:
           secretAsVolume:
           - mountPath: /mnt/my_vol
             secretName: my-secret
+            optional: False
diff --git a/kubernetes_platform/python/test/unit/test_config_map.py b/kubernetes_platform/python/test/unit/test_config_map.py
--- a/kubernetes_platform/python/test/unit/test_config_map.py
+++ b/kubernetes_platform/python/test/unit/test_config_map.py
@@ -38,7 +38,66 @@ def my_pipeline():
                             'exec-comp': {
                                 'configMapAsVolume': [{
                                     'configMapName': 'cm-name',
-                                    'mountPath': 'cmpath'
+                                    'mountPath': 'cmpath',
+                                    'optional': False
+                                }]
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+    def test_use_one_optional_true(self):
+
+        @dsl.pipeline
+        def my_pipeline():
+            task = comp()
+            kubernetes.use_config_map_as_volume(
+                task,
+                config_map_name='cm-name',
+                mount_path='cmpath',
+                optional=True)
+
+        assert json_format.MessageToDict(my_pipeline.platform_spec) == {
+            'platforms': {
+                'kubernetes': {
+                    'deploymentSpec': {
+                        'executors': {
+                            'exec-comp': {
+                                'configMapAsVolume': [{
+                                    'configMapName': 'cm-name',
+                                    'mountPath': 'cmpath',
+                                    'optional': True
+                                }]
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+    def test_use_one_optional_false(self):
+
+        @dsl.pipeline
+        def my_pipeline():
+            task = comp()
+            kubernetes.use_config_map_as_volume(
+                task,
+                config_map_name='cm-name',
+                mount_path='cmpath',
+                optional=False)
+
+        assert json_format.MessageToDict(my_pipeline.platform_spec) == {
+            'platforms': {
+                'kubernetes': {
+                    'deploymentSpec': {
+                        'executors': {
+                            'exec-comp': {
+                                'configMapAsVolume': [{
+                                    'configMapName': 'cm-name',
+                                    'mountPath': 'cmpath',
+                                    'optional': False
                                 }]
                             }
                         }
@@ -72,11 +131,13 @@ def my_pipeline():
                                 'configMapAsVolume': [
                                     {
                                         'configMapName': 'cm-name1',
-                                        'mountPath': 'cmpath1'
+                                        'mountPath': 'cmpath1',
+                                        'optional': False
                                     },
                                     {
                                         'configMapName': 'cm-name2',
-                                        'mountPath': 'cmpath2'
+                                        'mountPath': 'cmpath2',
+                                        'optional': False
                                     },
                                 ]
                             }
@@ -119,7 +180,8 @@ def my_pipeline():
                                 }],
                                 'configMapAsVolume': [{
                                     'configMapName': 'cm-name2',
-                                    'mountPath': 'cmpath2'
+                                    'mountPath': 'cmpath2',
+                                    'optional': False
                                 },]
                             }
                         }
@@ -156,7 +218,8 @@ def my_pipeline():
                                 }],
                                 'configMapAsVolume': [{
                                     'configMapName': 'cm-name',
-                                    'mountPath': 'cmpath'
+                                    'mountPath': 'cmpath',
+                                    'optional': False
                                 }]
                             }
                         }
@@ -289,7 +352,8 @@ def my_pipeline():
                                 }],
                                 'configMapAsVolume': [{
                                     'configMapName': 'cm-name2',
-                                    'mountPath': 'cmpath2'
+                                    'mountPath': 'cmpath2',
+                                    'optional': False
                                 },]
                             }
                         }
diff --git a/kubernetes_platform/python/test/unit/test_image_pull_secrets.py b/kubernetes_platform/python/test/unit/test_image_pull_secrets.py
--- a/kubernetes_platform/python/test/unit/test_image_pull_secrets.py
+++ b/kubernetes_platform/python/test/unit/test_image_pull_secrets.py
@@ -33,8 +33,7 @@ def my_pipeline():
                         'executors': {
                             'exec-comp': {
                                 'imagePullSecret': [{
-                                    'secretName':
-                                        'secret-name'
+                                    'secretName': 'secret-name'
                                 }]
                             }
                         }
@@ -48,7 +47,8 @@ def test_add_two(self):
         @dsl.pipeline
         def my_pipeline():
             task = comp()
-            kubernetes.set_image_pull_secrets(task, ['secret-name1', 'secret-name2'])
+            kubernetes.set_image_pull_secrets(task,
+                                              ['secret-name1', 'secret-name2'])
 
         assert json_format.MessageToDict(my_pipeline.platform_spec) == {
             'platforms': {
@@ -56,13 +56,13 @@ def my_pipeline():
                     'deploymentSpec': {
                         'executors': {
                             'exec-comp': {
-                                'imagePullSecret': [{
-                                    'secretName':
-                                        'secret-name1'
-                                }, {
-                                    'secretName':
-                                        'secret-name2'
-                                },
+                                'imagePullSecret': [
+                                    {
+                                        'secretName': 'secret-name1'
+                                    },
+                                    {
+                                        'secretName': 'secret-name2'
+                                    },
                                 ]
                             }
                         }
@@ -92,11 +92,11 @@ def my_pipeline():
                             'exec-comp': {
                                 'secretAsVolume': [{
                                     'secretName': 'secret-name',
-                                    'mountPath': '/mnt/my_vol'
+                                    'mountPath': '/mnt/my_vol',
+                                    'optional': False
                                 }],
                                 'imagePullSecret': [{
-                                    'secretName':
-                                        'secret-name'
+                                    'secretName': 'secret-name'
                                 }]
                             }
                         }
diff --git a/kubernetes_platform/python/test/unit/test_node_selector.py b/kubernetes_platform/python/test/unit/test_node_selector.py
--- a/kubernetes_platform/python/test/unit/test_node_selector.py
+++ b/kubernetes_platform/python/test/unit/test_node_selector.py
@@ -113,7 +113,8 @@ def my_pipeline():
                                 },
                                 'secretAsVolume': [{
                                     'secretName': 'my-secret',
-                                    'mountPath': '/mnt/my_vol'
+                                    'mountPath': '/mnt/my_vol',
+                                    'optional': False
                                 }]
                             }
                         }
diff --git a/kubernetes_platform/python/test/unit/test_pod_metadata.py b/kubernetes_platform/python/test/unit/test_pod_metadata.py
--- a/kubernetes_platform/python/test/unit/test_pod_metadata.py
+++ b/kubernetes_platform/python/test/unit/test_pod_metadata.py
@@ -157,7 +157,8 @@ def my_pipeline():
                                 },
                                 'secretAsVolume': [{
                                     'secretName': 'my-secret',
-                                    'mountPath': '/mnt/my_vol'
+                                    'mountPath': '/mnt/my_vol',
+                                    'optional': False
                                 }]
                             }
                         }
diff --git a/kubernetes_platform/python/test/unit/test_secret.py b/kubernetes_platform/python/test/unit/test_secret.py
--- a/kubernetes_platform/python/test/unit/test_secret.py
+++ b/kubernetes_platform/python/test/unit/test_secret.py
@@ -38,7 +38,66 @@ def my_pipeline():
                             'exec-comp': {
                                 'secretAsVolume': [{
                                     'secretName': 'secret-name',
-                                    'mountPath': 'secretpath'
+                                    'mountPath': 'secretpath',
+                                    'optional': False
+                                }]
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+    def test_use_one_optional_true(self):
+
+        @dsl.pipeline
+        def my_pipeline():
+            task = comp()
+            kubernetes.use_secret_as_volume(
+                task,
+                secret_name='secret-name',
+                mount_path='secretpath',
+                optional=True)
+
+        assert json_format.MessageToDict(my_pipeline.platform_spec) == {
+            'platforms': {
+                'kubernetes': {
+                    'deploymentSpec': {
+                        'executors': {
+                            'exec-comp': {
+                                'secretAsVolume': [{
+                                    'secretName': 'secret-name',
+                                    'mountPath': 'secretpath',
+                                    'optional': True
+                                }]
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+    def test_use_one_optional_false(self):
+
+        @dsl.pipeline
+        def my_pipeline():
+            task = comp()
+            kubernetes.use_secret_as_volume(
+                task,
+                secret_name='secret-name',
+                mount_path='secretpath',
+                optional=False)
+
+        assert json_format.MessageToDict(my_pipeline.platform_spec) == {
+            'platforms': {
+                'kubernetes': {
+                    'deploymentSpec': {
+                        'executors': {
+                            'exec-comp': {
+                                'secretAsVolume': [{
+                                    'secretName': 'secret-name',
+                                    'mountPath': 'secretpath',
+                                    'optional': False
                                 }]
                             }
                         }
@@ -72,11 +131,13 @@ def my_pipeline():
                                 'secretAsVolume': [
                                     {
                                         'secretName': 'secret-name1',
-                                        'mountPath': 'secretpath1'
+                                        'mountPath': 'secretpath1',
+                                        'optional': False
                                     },
                                     {
                                         'secretName': 'secret-name2',
-                                        'mountPath': 'secretpath2'
+                                        'mountPath': 'secretpath2',
+                                        'optional': False
                                     },
                                 ]
                             }
@@ -119,7 +180,8 @@ def my_pipeline():
                                 }],
                                 'secretAsVolume': [{
                                     'secretName': 'secret-name2',
-                                    'mountPath': 'secretpath2'
+                                    'mountPath': 'secretpath2',
+                                    'optional': False
                                 },]
                             }
                         }
@@ -156,7 +218,8 @@ def my_pipeline():
                                 }],
                                 'secretAsVolume': [{
                                     'secretName': 'secret-name',
-                                    'mountPath': 'secretpath'
+                                    'mountPath': 'secretpath',
+                                    'optional': False
                                 }]
                             }
                         }
@@ -289,7 +352,8 @@ def my_pipeline():
                                 }],
                                 'secretAsVolume': [{
                                     'secretName': 'secret-name2',
-                                    'mountPath': 'secretpath2'
+                                    'mountPath': 'secretpath2',
+                                    'optional': False
                                 },]
                             }
                         }
diff --git a/kubernetes_platform/python/test/unit/test_tolerations.py b/kubernetes_platform/python/test/unit/test_tolerations.py
--- a/kubernetes_platform/python/test/unit/test_tolerations.py
+++ b/kubernetes_platform/python/test/unit/test_tolerations.py
@@ -163,6 +163,7 @@ def my_pipeline():
                                 'secretAsVolume': [{
                                     'secretName': 'my-secret',
                                     'mountPath': '/mnt/my_vol',
+                                    'optional': False
                                 },],
                             },
                         }
diff --git a/kubernetes_platform/python/test/unit/test_volume.py b/kubernetes_platform/python/test/unit/test_volume.py
--- a/kubernetes_platform/python/test/unit/test_volume.py
+++ b/kubernetes_platform/python/test/unit/test_volume.py
@@ -156,7 +156,8 @@ def my_pipeline():
                                 }],
                                 'secretAsVolume': [{
                                     'secretName': 'secret-name',
-                                    'mountPath': 'secretpath'
+                                    'mountPath': 'secretpath',
+                                    'optional': False
                                 }]
                             }
                         }