Repository: GoogleCloudPlatform/spark-on-k8s-operator
Pull request #1647 (2022-11-18 15:28:38)

Added permissions for leader election #1635
Added new RBAC permissions needed by default for leader election for the coordination/v1 API. 
Required after upgrade to golang:1.19.2.
In k8s.io/client-go@v0.25.3/tools/leaderelection/resourcelock/interface.go:166 `configMapsResourceLock` was removed and should be replaced by `ConfigMapsLeasesResourceLock`.

https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/issues/1635

sairamankumar2: We are facing the following error and this PR will help resolve it. 

`error retrieving resource lock cirrus-spark-operator/spark-operator-lock: leases.coordination.k8s.io "spark-operator-lock" is forbidden: User "system:serviceaccount:cirrus-spark-operator:spark-operator" cannot get resource "leases" in API group "coordination.k8s.io" in the namespace "cirrus-spark-operator"`
sairamankumar2: @liyinan926 Can you kindly review this PR?
pdrastil: This is duplicate of https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/pull/1645. Does this work without granting full access to CMs? From leader election I've seen this typically required more than just create / get / update / delete as K8s controller is also watching for changes in this resources.

Patch:
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..b6b6a6c
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,16 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
diff --git a/charts/spark-operator-chart/Chart.yaml b/charts/spark-operator-chart/Chart.yaml
index 4fad2b5..da9a2e4 100644
--- a/charts/spark-operator-chart/Chart.yaml
+++ b/charts/spark-operator-chart/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: spark-operator
 description: A Helm chart for Spark on Kubernetes operator
-version: 1.1.26
+version: 1.1.27
 appVersion: v1beta2-1.3.8-3.1.1
 keywords:
   - spark
diff --git a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
index 3ee45fb..1d23c4d 100644
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
@@ -3936,6 +3936,29 @@ spec:
                                 pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                                 x-kubernetes-int-or-string: true
                             type: object
+                          ephemeral:
+                            properties:
+                              volumeClaimTemplate:
+                                properties:
+                                  spec:
+                                    properties:
+                                      accessModes:
+                                        items:
+                                          type: string
+                                        type: array
+                                      resources:
+                                        properties:
+                                          requests:
+                                            properties:
+                                              storage:
+                                                type: string
+                                            type: object
+                                        type: object
+                                      storageClassName:
+                                        type: string
+                                    type: object
+                                type: object
+                            type: object
                           fc:
                             properties:
                               fsType:
diff --git a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
index fbb52e2..5ca3100 100644
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
@@ -3924,6 +3924,29 @@ spec:
                             pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                             x-kubernetes-int-or-string: true
                         type: object
+                      ephemeral:
+                        properties:
+                          volumeClaimTemplate:
+                            properties:
+                              spec:
+                                properties:
+                                  accessModes:
+                                    items:
+                                      type: string
+                                    type: array
+                                  resources:
+                                    properties:
+                                      requests:
+                                        properties:
+                                          storage:
+                                            type: string
+                                        type: object
+                                    type: object
+                                  storageClassName:
+                                    type: string
+                                type: object
+                            type: object
+                        type: object
                       fc:
                         properties:
                           fsType:
diff --git a/charts/spark-operator-chart/templates/rbac.yaml b/charts/spark-operator-chart/templates/rbac.yaml
index 56ddc1e..6f5d97c 100644
--- a/charts/spark-operator-chart/templates/rbac.yaml
+++ b/charts/spark-operator-chart/templates/rbac.yaml
@@ -105,7 +105,25 @@ rules:
   verbs:
   - delete
   {{- end }}
-
+  {{- if gt (int .Values.replicaCount) 1 }}
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  resourceNames:
+  - {{ .Values.leaderElection.lockName }}
+  verbs:
+  - get
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  {{- end }}
 ---
 
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/charts/spark-operator-chart/templates/webhook-cleanup-job.yaml b/charts/spark-operator-chart/templates/webhook-cleanup-job.yaml
index 0ab1350..dadd1df 100644
--- a/charts/spark-operator-chart/templates/webhook-cleanup-job.yaml
+++ b/charts/spark-operator-chart/templates/webhook-cleanup-job.yaml
@@ -45,7 +45,6 @@ spec:
           -H \"Content-Type: application/json\" \
           --data \"{\\\"kind\\\":\\\"DeleteOptions\\\",\\\"apiVersion\\\":\\\"batch/v1\\\",\\\"propagationPolicy\\\":\\\"Foreground\\\"}\" \
           https://kubernetes.default.svc/apis/batch/v1/namespaces/{{ .Release.Namespace }}/jobs/{{ include "spark-operator.fullname" . }}-webhook-init"
-{{ end }}
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
@@ -54,3 +53,4 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{ end }}
diff --git a/charts/spark-operator-chart/templates/webhook-init-job.yaml b/charts/spark-operator-chart/templates/webhook-init-job.yaml
index 10fab8f..e944839 100644
--- a/charts/spark-operator-chart/templates/webhook-init-job.yaml
+++ b/charts/spark-operator-chart/templates/webhook-init-job.yaml
@@ -35,7 +35,6 @@ spec:
             "-r", "{{ include "spark-operator.fullname" . }}-webhook-certs",
             "-p"
           ]
-{{ end }}
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
@@ -44,3 +43,4 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{ end }}
diff --git a/docs/user-guide.md b/docs/user-guide.md
index 3cc65ef..1aeeb12 100644
--- a/docs/user-guide.md
+++ b/docs/user-guide.md
@@ -614,7 +614,7 @@ spec:
 ```
 
 ### Using Container LifeCycle Hooks
-A Spark Application can optionally specify a [Container Lifecycle Hooks](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks) for a driver. It is useful in cases where you need a PreStop or PostStart hooks to driver.
+A Spark Application can optionally specify a [Container Lifecycle Hooks](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks) for a driver. It is useful in cases where you need a PreStop or PostStart hooks to driver and executor.
 
 ```yaml
 spec:
diff --git a/docs/who-is-using.md b/docs/who-is-using.md
index 2a6167e..76a609d 100644
--- a/docs/who-is-using.md
+++ b/docs/who-is-using.md
@@ -39,3 +39,4 @@
 | [DiDi](https://www.didiglobal.com) | @Run-Lin | Evaluation | Data Infrastructure |
 | [DeepCure](https://www.deepcure.ai) | @mschroering | Production | Spark / ML |
 | [C2FO](https://www.c2fo.com/) | @vanhoale | Production | Data Platform / Data Infrastructure |
+| [Kognita](https://kognita.com.br/) | @andreclaudino | Production | MLOps, Data Platform / Data Infrastructure, ML/AI |
diff --git a/main.go b/main.go
index 2bb5496..db4861b 100644
--- a/main.go
+++ b/main.go
@@ -219,7 +219,11 @@ func main() {
 
 	if *enableLeaderElection {
 		glog.Info("Waiting to be elected leader before starting application controller goroutines")
-		<-startCh
+		select {
+		case <-signalCh:
+			os.Exit(0)
+		case <-startCh:
+		}
 	}
 
 	glog.Info("Starting application controller goroutines")
diff --git a/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go b/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
index bd4aee2..2c89fe9 100644
--- a/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
+++ b/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
@@ -580,6 +580,9 @@ type ExecutorSpec struct {
 	// GC settings or other logging.
 	// +optional
 	JavaOptions *string `json:"javaOptions,omitempty"`
+	// Lifecycle for running preStop or postStart commands
+	// +optional
+	Lifecycle *apiv1.Lifecycle `json:"lifecycle,omitempty"`
 	// DeleteOnTermination specify whether executor pods should be deleted in case of failure or normal termination.
 	// Maps to `spark.kubernetes.executor.deleteOnTermination` that is available since Spark 3.0.
 	// +optional
diff --git a/pkg/webhook/patch.go b/pkg/webhook/patch.go
index 3034ebd..bc9a748 100644
--- a/pkg/webhook/patch.go
+++ b/pkg/webhook/patch.go
@@ -765,8 +765,13 @@ func addTerminationGracePeriodSeconds(pod *corev1.Pod, app *v1beta2.SparkApplica
 
 func addPodLifeCycleConfig(pod *corev1.Pod, app *v1beta2.SparkApplication) *patchOperation {
 	var lifeCycle *corev1.Lifecycle
+	var containerName string
 	if util.IsDriverPod(pod) {
 		lifeCycle = app.Spec.Driver.Lifecycle
+		containerName = config.SparkDriverContainerName
+	} else if util.IsExecutorPod(pod) {
+		lifeCycle = app.Spec.Executor.Lifecycle
+		containerName = config.SparkExecutorContainerName
 	}
 	if lifeCycle == nil {
 		return nil
@@ -775,12 +780,12 @@ func addPodLifeCycleConfig(pod *corev1.Pod, app *v1beta2.SparkApplication) *patc
 	i := 0
 	// Find the driver container in the pod.
 	for ; i < len(pod.Spec.Containers); i++ {
-		if pod.Spec.Containers[i].Name == config.SparkDriverContainerName {
+		if pod.Spec.Containers[i].Name == containerName {
 			break
 		}
 	}
 	if i == len(pod.Spec.Containers) {
-		glog.Warningf("Spark driver container not found in pod %s", pod.Name)
+		glog.Warningf("Spark container %s not found in pod %s", containerName, pod.Name)
 		return nil
 	}
 


Test patch:
diff --git a/pkg/webhook/patch_test.go b/pkg/webhook/patch_test.go
index 9efa740..4b66b62 100644
--- a/pkg/webhook/patch_test.go
+++ b/pkg/webhook/patch_test.go
@@ -1752,6 +1752,9 @@ func TestPatchSparkPod_Lifecycle(t *testing.T) {
 	preStopTest := &corev1.ExecAction{
 		Command: []string{"/bin/sh", "-c", "echo Hello from the pre stop handler > /usr/share/message"},
 	}
+	postStartTest := &corev1.ExecAction{
+		Command: []string{"/bin/sh", "-c", "echo Hello from the post start handler > /usr/share/message"},
+	}
 	app := &v1beta2.SparkApplication{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "spark-test",
@@ -1763,6 +1766,11 @@ func TestPatchSparkPod_Lifecycle(t *testing.T) {
 					PreStop: &corev1.LifecycleHandler{Exec: preStopTest},
 				},
 			},
+			Executor: v1beta2.ExecutorSpec{
+				Lifecycle: &corev1.Lifecycle{
+					PostStart: &corev1.LifecycleHandler{Exec: postStartTest},
+				},
+			},
 		},
 	}
 
@@ -1784,11 +1792,34 @@ func TestPatchSparkPod_Lifecycle(t *testing.T) {
 		},
 	}
 
+	executorPod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "spark-executor",
+			Labels: map[string]string{
+				config.SparkRoleLabel:               config.SparkExecutorRole,
+				config.LaunchedBySparkOperatorLabel: "true",
+			},
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:  config.SparkExecutorContainerName,
+					Image: "spark-executor:latest",
+				},
+			},
+		},
+	}
+
 	modifiedDriverPod, err := getModifiedPod(driverPod, app)
 	if err != nil {
 		t.Fatal(err)
 	}
+	modifiedExecutorPod, err := getModifiedPod(executorPod, app)
+	if err != nil {
+		t.Fatal(err)
+	}
 	assert.Equal(t, preStopTest, modifiedDriverPod.Spec.Containers[0].Lifecycle.PreStop.Exec)
+	assert.Equal(t, postStartTest, modifiedExecutorPod.Spec.Containers[0].Lifecycle.PostStart.Exec)
 }
 
 func getModifiedPod(pod *corev1.Pod, app *v1beta2.SparkApplication) (*corev1.Pod, error) {