Repository: GoogleCloudPlatform/spark-on-k8s-operator
Pull request #1262 (2021-05-17 14:38:04)

implement shareProcessNamespace in SparkPodSpec
Hello,

This PR implement `shareProcessNamespace` in `SparkPodSpec`

Don't hesitate to tell me if i need to change anything

liyinan926: Looks like you also need to update the API doc following instruction in https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/developer-guide.md#build-the-api-specification-doc.
ImpSy: @liyinan926 is this ok to merge for you ?
ImpSy: Fixes are done

Patch:
diff --git a/charts/spark-operator-chart/Chart.yaml b/charts/spark-operator-chart/Chart.yaml
index 0258a52..a701432 100644
--- a/charts/spark-operator-chart/Chart.yaml
+++ b/charts/spark-operator-chart/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: spark-operator
 description: A Helm chart for Spark on Kubernetes operator
-version: 1.1.1
+version: 1.1.2
 appVersion: v1beta2-1.2.3-3.1.1
 keywords:
   - spark
diff --git a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
index 5e28c26..863d984 100644
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
@@ -1305,6 +1305,8 @@ spec:
                       additionalProperties:
                         type: string
                       type: object
+                    shareProcessNamespace:
+                      type: boolean
                     sidecars:
                       items:
                         properties:
@@ -3007,6 +3009,8 @@ spec:
                       type: object
                     serviceAccount:
                       type: string
+                    shareProcessNamespace:
+                      type: boolean
                     sidecars:
                       items:
                         properties:
@@ -3709,6 +3713,8 @@ spec:
                     servicePort:
                       format: int32
                       type: integer
+                    serviceType:
+                      type: string
                   type: object
                 sparkVersion:
                   type: string
diff --git a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
index 98ed07f..68abbf0 100644
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
@@ -1291,6 +1291,8 @@ spec:
                   additionalProperties:
                     type: string
                   type: object
+                shareProcessNamespace:
+                  type: boolean
                 sidecars:
                   items:
                     properties:
@@ -2993,6 +2995,8 @@ spec:
                   type: object
                 serviceAccount:
                   type: string
+                shareProcessNamespace:
+                  type: boolean
                 sidecars:
                   items:
                     properties:
@@ -3695,6 +3699,10 @@ spec:
                 servicePort:
                   format: int32
                   type: integer
+                servicePortName:
+                  type: string
+                serviceType:
+                  type: string
               type: object
             sparkVersion:
               type: string
diff --git a/docs/api-docs.md b/docs/api-docs.md
index 5a25a6e..0eaad6a 100644
--- a/docs/api-docs.md
+++ b/docs/api-docs.md
@@ -2884,6 +2884,18 @@ string
 <p>HostAliases settings for the pod, following the Kubernetes specifications.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>shareProcessNamespace</code></br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>ShareProcessNamespace settings for the pod, following the Kubernetes specifications.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="sparkoperator.k8s.io/v1beta2.SparkUIConfiguration">SparkUIConfiguration
@@ -2918,7 +2930,7 @@ TargetPort should be the same as the one defined in spark.ui.port</p>
 </tr>
 <tr>
 <td>
-<code>ServicePortName</code></br>
+<code>servicePortName</code></br>
 <em>
 string
 </em>
@@ -2975,5 +2987,5 @@ map[string]string
 <hr/>
 <p><em>
 Generated with <code>gen-crd-api-reference-docs</code>
-on git commit <code>eb07ee0</code>.
+on git commit <code>e616816</code>.
 </em></p>
diff --git a/manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml b/manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
index 15348c5..863d984 100644
--- a/manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+++ b/manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
@@ -5,7 +5,6 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: (unknown)
-  creationTimestamp: null
   name: scheduledsparkapplications.sparkoperator.k8s.io
 spec:
   group: sparkoperator.k8s.io
@@ -508,6 +507,17 @@ spec:
                       - name
                       - quantity
                       type: object
+                    hostAliases:
+                      items:
+                        properties:
+                          hostnames:
+                            items:
+                              type: string
+                            type: array
+                          ip:
+                            type: string
+                        type: object
+                      type: array
                     hostNetwork:
                       type: boolean
                     image:
@@ -1295,6 +1305,8 @@ spec:
                       additionalProperties:
                         type: string
                       type: object
+                    shareProcessNamespace:
+                      type: boolean
                     sidecars:
                       items:
                         properties:
@@ -2307,6 +2319,17 @@ spec:
                       - name
                       - quantity
                       type: object
+                    hostAliases:
+                      items:
+                        properties:
+                          hostnames:
+                            items:
+                              type: string
+                            type: array
+                          ip:
+                            type: string
+                        type: object
+                      type: array
                     hostNetwork:
                       type: boolean
                     image:
@@ -2986,6 +3009,8 @@ spec:
                       type: object
                     serviceAccount:
                       type: string
+                    shareProcessNamespace:
+                      type: boolean
                     sidecars:
                       items:
                         properties:
@@ -3614,6 +3639,8 @@ spec:
                           maximum: 49151
                           minimum: 1024
                           type: integer
+                        portName:
+                          type: string
                       required:
                       - jmxExporterJar
                       type: object
diff --git a/manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml b/manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml
index 8032f79..68abbf0 100644
--- a/manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml
+++ b/manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml
@@ -5,7 +5,6 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: (unknown)
-  creationTimestamp: null
   name: sparkapplications.sparkoperator.k8s.io
 spec:
   group: sparkoperator.k8s.io
@@ -494,6 +493,17 @@ spec:
                   - name
                   - quantity
                   type: object
+                hostAliases:
+                  items:
+                    properties:
+                      hostnames:
+                        items:
+                          type: string
+                        type: array
+                      ip:
+                        type: string
+                    type: object
+                  type: array
                 hostNetwork:
                   type: boolean
                 image:
@@ -1281,6 +1291,8 @@ spec:
                   additionalProperties:
                     type: string
                   type: object
+                shareProcessNamespace:
+                  type: boolean
                 sidecars:
                   items:
                     properties:
@@ -2293,6 +2305,17 @@ spec:
                   - name
                   - quantity
                   type: object
+                hostAliases:
+                  items:
+                    properties:
+                      hostnames:
+                        items:
+                          type: string
+                        type: array
+                      ip:
+                        type: string
+                    type: object
+                  type: array
                 hostNetwork:
                   type: boolean
                 image:
@@ -2972,6 +2995,8 @@ spec:
                   type: object
                 serviceAccount:
                   type: string
+                shareProcessNamespace:
+                  type: boolean
                 sidecars:
                   items:
                     properties:
@@ -3600,6 +3625,8 @@ spec:
                       maximum: 49151
                       minimum: 1024
                       type: integer
+                    portName:
+                      type: string
                   required:
                   - jmxExporterJar
                   type: object
diff --git a/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go b/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
index 83376b2..618e5ba 100644
--- a/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
+++ b/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
@@ -523,6 +523,9 @@ type SparkPodSpec struct {
 	// HostAliases settings for the pod, following the Kubernetes specifications.
 	// +optional
 	HostAliases []apiv1.HostAlias `json:"hostAliases,omitempty"`
+	// ShareProcessNamespace settings for the pod, following the Kubernetes specifications.
+	// +optional
+	ShareProcessNamespace *bool `json:"shareProcessNamespace,omitempty"`
 }
 
 // DriverSpec is specification of the driver.
diff --git a/pkg/apis/sparkoperator.k8s.io/v1beta2/zz_generated.deepcopy.go b/pkg/apis/sparkoperator.k8s.io/v1beta2/zz_generated.deepcopy.go
index 662b276..7c5a4d9 100644
--- a/pkg/apis/sparkoperator.k8s.io/v1beta2/zz_generated.deepcopy.go
+++ b/pkg/apis/sparkoperator.k8s.io/v1beta2/zz_generated.deepcopy.go
@@ -175,6 +175,11 @@ func (in *DriverSpec) DeepCopyInto(out *DriverSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.Ports != nil {
+		in, out := &in.Ports, &out.Ports
+		*out = make([]Port, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -248,6 +253,11 @@ func (in *ExecutorSpec) DeepCopyInto(out *ExecutorSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.Ports != nil {
+		in, out := &in.Ports, &out.Ports
+		*out = make([]Port, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -340,6 +350,22 @@ func (in *NamePath) DeepCopy() *NamePath {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Port) DeepCopyInto(out *Port) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Port.
+func (in *Port) DeepCopy() *Port {
+	if in == nil {
+		return nil
+	}
+	out := new(Port)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) {
 	*out = *in
@@ -947,6 +973,11 @@ func (in *SparkPodSpec) DeepCopyInto(out *SparkPodSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.ShareProcessNamespace != nil {
+		in, out := &in.ShareProcessNamespace, &out.ShareProcessNamespace
+		*out = new(bool)
+		**out = **in
+	}
 	return
 }
 
@@ -968,6 +999,11 @@ func (in *SparkUIConfiguration) DeepCopyInto(out *SparkUIConfiguration) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.ServicePortName != nil {
+		in, out := &in.ServicePortName, &out.ServicePortName
+		*out = new(string)
+		**out = **in
+	}
 	if in.ServiceType != nil {
 		in, out := &in.ServiceType, &out.ServiceType
 		*out = new(v1.ServiceType)
diff --git a/pkg/webhook/patch.go b/pkg/webhook/patch.go
index 870122e..dea305e 100644
--- a/pkg/webhook/patch.go
+++ b/pkg/webhook/patch.go
@@ -103,6 +103,11 @@ func patchSparkPod(pod *corev1.Pod, app *v1beta2.SparkApplication) []patchOperat
 		patchOps = append(patchOps, *op)
 	}
 
+	op = addShareProcessNamespace(pod, app)
+	if op != nil {
+		patchOps = append(patchOps, *op)
+	}
+
 	return patchOps
 }
 
@@ -834,3 +839,18 @@ func addHostAliases(pod *corev1.Pod, app *v1beta2.SparkApplication) []patchOpera
 	}
 	return ops
 }
+
+func addShareProcessNamespace(pod *corev1.Pod, app *v1beta2.SparkApplication) *patchOperation {
+	var shareProcessNamespace *bool
+	if util.IsDriverPod(pod) {
+		shareProcessNamespace = app.Spec.Driver.ShareProcessNamespace
+	}
+	if util.IsExecutorPod(pod) {
+		shareProcessNamespace = app.Spec.Executor.ShareProcessNamespace
+	}
+
+	if shareProcessNamespace == nil || *shareProcessNamespace == false {
+		return nil
+	}
+	return &patchOperation{Op: "add", Path: "/spec/shareProcessNamespace", Value: *shareProcessNamespace}
+}


Test patch:
diff --git a/pkg/webhook/patch_test.go b/pkg/webhook/patch_test.go
index b68585c..8856f16 100644
--- a/pkg/webhook/patch_test.go
+++ b/pkg/webhook/patch_test.go
@@ -1998,3 +1998,87 @@ func TestPatchSparkPod_Ports(t *testing.T) {
 	assert.Equal(t, int32(8082), modifiedExecutorPod.Spec.Containers[0].Ports[0].ContainerPort)
 	assert.Equal(t, int32(8083), modifiedExecutorPod.Spec.Containers[0].Ports[1].ContainerPort)
 }
+
+func TestPatchSparkPod_ShareProcessNamespace(t *testing.T) {
+	app := &v1beta2.SparkApplication{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "spark-test",
+			UID:  "spark-test-1",
+		},
+		Spec: v1beta2.SparkApplicationSpec{
+			Driver: v1beta2.DriverSpec{
+				SparkPodSpec: v1beta2.SparkPodSpec{},
+			},
+			Executor: v1beta2.ExecutorSpec{
+				SparkPodSpec: v1beta2.SparkPodSpec{},
+			},
+		},
+	}
+
+	var shareProcessNamespaceTrue = true
+	var shareProcessNamespaceFalse = false
+	tests := []*bool{
+		nil,
+		&shareProcessNamespaceTrue,
+		&shareProcessNamespaceFalse,
+	}
+
+	for _, test := range tests {
+
+		app.Spec.Driver.ShareProcessNamespace = test
+		app.Spec.Executor.ShareProcessNamespace = test
+		driverPod := &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "spark-driver",
+				Labels: map[string]string{
+					config.SparkRoleLabel:               config.SparkDriverRole,
+					config.LaunchedBySparkOperatorLabel: "true",
+				},
+			},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name:  config.SparkDriverContainerName,
+						Image: "spark-driver:latest",
+					},
+				},
+			},
+		}
+
+		executorPod := &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "spark-executor",
+				Labels: map[string]string{
+					config.SparkRoleLabel:               config.SparkExecutorRole,
+					config.LaunchedBySparkOperatorLabel: "true",
+				},
+			},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name:  config.SparkExecutorContainerName,
+						Image: "spark-executor:latest",
+					},
+				},
+			},
+		}
+
+		modifiedDriverPod, err := getModifiedPod(driverPod, app)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		modifiedExecutorPod, err := getModifiedPod(executorPod, app)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if test == nil || *test == false {
+			assert.Nil(t, modifiedDriverPod.Spec.ShareProcessNamespace)
+			assert.Nil(t, modifiedExecutorPod.Spec.ShareProcessNamespace)
+		} else {
+			assert.Equal(t, true, *modifiedDriverPod.Spec.ShareProcessNamespace)
+			assert.Equal(t, true, *modifiedExecutorPod.Spec.ShareProcessNamespace)
+		}
+	}
+}