Repository: GoogleCloudPlatform/spark-on-k8s-operator
Pull request #1669 (2023-01-12 15:01:22)

Extra helm chart labels
Added option to set extra common labels
Added options to set podlabels on the pods of the cleanup and init jobs


Patch:
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..b6b6a6c
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,16 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
diff --git a/charts/spark-operator-chart/Chart.yaml b/charts/spark-operator-chart/Chart.yaml
index 4fad2b5..da9a2e4 100644
--- a/charts/spark-operator-chart/Chart.yaml
+++ b/charts/spark-operator-chart/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: spark-operator
 description: A Helm chart for Spark on Kubernetes operator
-version: 1.1.26
+version: 1.1.27
 appVersion: v1beta2-1.3.8-3.1.1
 keywords:
   - spark
diff --git a/charts/spark-operator-chart/README.md b/charts/spark-operator-chart/README.md
index c00d9aa..2314c82 100644
--- a/charts/spark-operator-chart/README.md
+++ b/charts/spark-operator-chart/README.md
@@ -81,6 +81,7 @@ All charts linted successfully
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Affinity for pod assignment |
 | batchScheduler.enable | bool | `false` | Enable batch scheduler for spark jobs scheduling. If enabled, users can specify batch scheduler name in spark application |
+| commonLabels | object | `{}` | Common labels to add to the resources |
 | controllerThreads | int | `10` | Operator concurrency, higher values might increase memory usage |
 | fullnameOverride | string | `""` | String to override release name |
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
@@ -126,8 +127,10 @@ All charts linted successfully
 | tolerations | list | `[]` | List of node taints to tolerate |
 | uiService.enable | bool | `true` | Enable UI service creation for Spark application |
 | webhook.cleanupAnnotations | object | `{"helm.sh/hook":"pre-delete, pre-upgrade","helm.sh/hook-delete-policy":"hook-succeeded"}` | The annotations applied to the cleanup job, required for helm lifecycle hooks |
+| webhook.cleanupPodLabels | object | `{}` | The podLabels applied to the pod of the cleanup job |
 | webhook.enable | bool | `false` | Enable webhook server |
 | webhook.initAnnotations | object | `{"helm.sh/hook":"pre-install, pre-upgrade","helm.sh/hook-weight":"50"}` | The annotations applied to init job, required to restore certs deleted by the cleanup job during upgrade |
+| webhook.initPodLabels | object | `{}` | The podLabels applied to the pod of the init job |
 | webhook.namespaceSelector | string | `""` | The webhook server will only operate on namespaces with this label, specified in the form key1=value1,key2=value2. Empty string (default) will operate on all namespaces |
 | webhook.port | int | `8080` | Webhook service port |
 | webhook.timeout | int | `30` |  |
diff --git a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
index 3ee45fb..1d23c4d 100644
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
@@ -3936,6 +3936,29 @@ spec:
                                 pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                                 x-kubernetes-int-or-string: true
                             type: object
+                          ephemeral:
+                            properties:
+                              volumeClaimTemplate:
+                                properties:
+                                  spec:
+                                    properties:
+                                      accessModes:
+                                        items:
+                                          type: string
+                                        type: array
+                                      resources:
+                                        properties:
+                                          requests:
+                                            properties:
+                                              storage:
+                                                type: string
+                                            type: object
+                                        type: object
+                                      storageClassName:
+                                        type: string
+                                    type: object
+                                type: object
+                            type: object
                           fc:
                             properties:
                               fsType:
diff --git a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
index fbb52e2..5ca3100 100644
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
@@ -3924,6 +3924,29 @@ spec:
                             pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                             x-kubernetes-int-or-string: true
                         type: object
+                      ephemeral:
+                        properties:
+                          volumeClaimTemplate:
+                            properties:
+                              spec:
+                                properties:
+                                  accessModes:
+                                    items:
+                                      type: string
+                                    type: array
+                                  resources:
+                                    properties:
+                                      requests:
+                                        properties:
+                                          storage:
+                                            type: string
+                                        type: object
+                                    type: object
+                                  storageClassName:
+                                    type: string
+                                type: object
+                            type: object
+                        type: object
                       fc:
                         properties:
                           fsType:
diff --git a/charts/spark-operator-chart/templates/_helpers.tpl b/charts/spark-operator-chart/templates/_helpers.tpl
index 2954e89..8e884ee 100644
--- a/charts/spark-operator-chart/templates/_helpers.tpl
+++ b/charts/spark-operator-chart/templates/_helpers.tpl
@@ -37,6 +37,9 @@ Common labels
 {{- define "spark-operator.labels" -}}
 helm.sh/chart: {{ include "spark-operator.chart" . }}
 {{ include "spark-operator.selectorLabels" . }}
+{{- if .Values.commonLabels }}
+{{ toYaml .Values.commonLabels }}
+{{- end }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
diff --git a/charts/spark-operator-chart/templates/rbac.yaml b/charts/spark-operator-chart/templates/rbac.yaml
index 56ddc1e..6f5d97c 100644
--- a/charts/spark-operator-chart/templates/rbac.yaml
+++ b/charts/spark-operator-chart/templates/rbac.yaml
@@ -105,7 +105,25 @@ rules:
   verbs:
   - delete
   {{- end }}
-
+  {{- if gt (int .Values.replicaCount) 1 }}
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  resourceNames:
+  - {{ .Values.leaderElection.lockName }}
+  verbs:
+  - get
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  {{- end }}
 ---
 
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/charts/spark-operator-chart/templates/webhook-cleanup-job.yaml b/charts/spark-operator-chart/templates/webhook-cleanup-job.yaml
index 0ab1350..a6f2c1e 100644
--- a/charts/spark-operator-chart/templates/webhook-cleanup-job.yaml
+++ b/charts/spark-operator-chart/templates/webhook-cleanup-job.yaml
@@ -15,6 +15,10 @@ spec:
       annotations:
         "sidecar.istio.io/inject": "false"
       {{- end }}
+      {{- if .Values.webhook.cleanupPodLabels }}
+      labels:
+        {{- toYaml .Values.webhook.cleanupPodLabels | nindent 8 }}
+      {{- end }}
     spec:
       serviceAccountName: {{ include "spark-operator.serviceAccountName" . }}
       restartPolicy: OnFailure
@@ -45,7 +49,6 @@ spec:
           -H \"Content-Type: application/json\" \
           --data \"{\\\"kind\\\":\\\"DeleteOptions\\\",\\\"apiVersion\\\":\\\"batch/v1\\\",\\\"propagationPolicy\\\":\\\"Foreground\\\"}\" \
           https://kubernetes.default.svc/apis/batch/v1/namespaces/{{ .Release.Namespace }}/jobs/{{ include "spark-operator.fullname" . }}-webhook-init"
-{{ end }}
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
@@ -54,3 +57,4 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{ end }}
diff --git a/charts/spark-operator-chart/templates/webhook-init-job.yaml b/charts/spark-operator-chart/templates/webhook-init-job.yaml
index 10fab8f..009879d 100644
--- a/charts/spark-operator-chart/templates/webhook-init-job.yaml
+++ b/charts/spark-operator-chart/templates/webhook-init-job.yaml
@@ -15,6 +15,10 @@ spec:
       annotations:
         "sidecar.istio.io/inject": "false"
       {{- end }}
+      {{- if .Values.webhook.initPodLabels }}
+      labels:
+        {{- toYaml .Values.webhook.initPodLabels | nindent 8 }}
+      {{- end }}
     spec:
       serviceAccountName: {{ include "spark-operator.serviceAccountName" . }}
       restartPolicy: OnFailure
@@ -35,7 +39,6 @@ spec:
             "-r", "{{ include "spark-operator.fullname" . }}-webhook-certs",
             "-p"
           ]
-{{ end }}
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
@@ -44,3 +47,4 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{ end }}
diff --git a/charts/spark-operator-chart/values.yaml b/charts/spark-operator-chart/values.yaml
index 891b45f..5007b7e 100644
--- a/charts/spark-operator-chart/values.yaml
+++ b/charts/spark-operator-chart/values.yaml
@@ -2,6 +2,9 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
+# -- Common labels to add to the resources
+commonLabels: {}
+
 # replicaCount -- Desired number of pods, leaderElection will be enabled
 # if this is greater than 1
 replicaCount: 1
@@ -92,11 +95,15 @@ webhook:
   initAnnotations:
     "helm.sh/hook": pre-install, pre-upgrade
     "helm.sh/hook-weight": "50"
+  # -- The podLabels applied to the pod of the init job
+  initPodLabels: {}
   # -- The annotations applied to the cleanup job, required for helm lifecycle hooks
   cleanupAnnotations:
     "helm.sh/hook": pre-delete, pre-upgrade
     "helm.sh/hook-delete-policy": hook-succeeded
     # -- Webhook Timeout in seconds
+  # -- The podLabels applied to the pod of the cleanup job
+  cleanupPodLabels: {}
   timeout: 30
 
 metrics:
diff --git a/docs/user-guide.md b/docs/user-guide.md
index 3cc65ef..1aeeb12 100644
--- a/docs/user-guide.md
+++ b/docs/user-guide.md
@@ -614,7 +614,7 @@ spec:
 ```
 
 ### Using Container LifeCycle Hooks
-A Spark Application can optionally specify a [Container Lifecycle Hooks](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks) for a driver. It is useful in cases where you need a PreStop or PostStart hooks to driver.
+A Spark Application can optionally specify a [Container Lifecycle Hooks](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks) for a driver. It is useful in cases where you need a PreStop or PostStart hooks to driver and executor.
 
 ```yaml
 spec:
diff --git a/docs/who-is-using.md b/docs/who-is-using.md
index 2a6167e..76a609d 100644
--- a/docs/who-is-using.md
+++ b/docs/who-is-using.md
@@ -39,3 +39,4 @@
 | [DiDi](https://www.didiglobal.com) | @Run-Lin | Evaluation | Data Infrastructure |
 | [DeepCure](https://www.deepcure.ai) | @mschroering | Production | Spark / ML |
 | [C2FO](https://www.c2fo.com/) | @vanhoale | Production | Data Platform / Data Infrastructure |
+| [Kognita](https://kognita.com.br/) | @andreclaudino | Production | MLOps, Data Platform / Data Infrastructure, ML/AI |
diff --git a/main.go b/main.go
index 2bb5496..db4861b 100644
--- a/main.go
+++ b/main.go
@@ -219,7 +219,11 @@ func main() {
 
 	if *enableLeaderElection {
 		glog.Info("Waiting to be elected leader before starting application controller goroutines")
-		<-startCh
+		select {
+		case <-signalCh:
+			os.Exit(0)
+		case <-startCh:
+		}
 	}
 
 	glog.Info("Starting application controller goroutines")
diff --git a/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go b/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
index bd4aee2..2c89fe9 100644
--- a/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
+++ b/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
@@ -580,6 +580,9 @@ type ExecutorSpec struct {
 	// GC settings or other logging.
 	// +optional
 	JavaOptions *string `json:"javaOptions,omitempty"`
+	// Lifecycle for running preStop or postStart commands
+	// +optional
+	Lifecycle *apiv1.Lifecycle `json:"lifecycle,omitempty"`
 	// DeleteOnTermination specify whether executor pods should be deleted in case of failure or normal termination.
 	// Maps to `spark.kubernetes.executor.deleteOnTermination` that is available since Spark 3.0.
 	// +optional
diff --git a/pkg/webhook/patch.go b/pkg/webhook/patch.go
index 3034ebd..bc9a748 100644
--- a/pkg/webhook/patch.go
+++ b/pkg/webhook/patch.go
@@ -765,8 +765,13 @@ func addTerminationGracePeriodSeconds(pod *corev1.Pod, app *v1beta2.SparkApplica
 
 func addPodLifeCycleConfig(pod *corev1.Pod, app *v1beta2.SparkApplication) *patchOperation {
 	var lifeCycle *corev1.Lifecycle
+	var containerName string
 	if util.IsDriverPod(pod) {
 		lifeCycle = app.Spec.Driver.Lifecycle
+		containerName = config.SparkDriverContainerName
+	} else if util.IsExecutorPod(pod) {
+		lifeCycle = app.Spec.Executor.Lifecycle
+		containerName = config.SparkExecutorContainerName
 	}
 	if lifeCycle == nil {
 		return nil
@@ -775,12 +780,12 @@ func addPodLifeCycleConfig(pod *corev1.Pod, app *v1beta2.SparkApplication) *patc
 	i := 0
 	// Find the driver container in the pod.
 	for ; i < len(pod.Spec.Containers); i++ {
-		if pod.Spec.Containers[i].Name == config.SparkDriverContainerName {
+		if pod.Spec.Containers[i].Name == containerName {
 			break
 		}
 	}
 	if i == len(pod.Spec.Containers) {
-		glog.Warningf("Spark driver container not found in pod %s", pod.Name)
+		glog.Warningf("Spark container %s not found in pod %s", containerName, pod.Name)
 		return nil
 	}
 


Test patch:
diff --git a/pkg/webhook/patch_test.go b/pkg/webhook/patch_test.go
index 9efa740..4b66b62 100644
--- a/pkg/webhook/patch_test.go
+++ b/pkg/webhook/patch_test.go
@@ -1752,6 +1752,9 @@ func TestPatchSparkPod_Lifecycle(t *testing.T) {
 	preStopTest := &corev1.ExecAction{
 		Command: []string{"/bin/sh", "-c", "echo Hello from the pre stop handler > /usr/share/message"},
 	}
+	postStartTest := &corev1.ExecAction{
+		Command: []string{"/bin/sh", "-c", "echo Hello from the post start handler > /usr/share/message"},
+	}
 	app := &v1beta2.SparkApplication{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "spark-test",
@@ -1763,6 +1766,11 @@ func TestPatchSparkPod_Lifecycle(t *testing.T) {
 					PreStop: &corev1.LifecycleHandler{Exec: preStopTest},
 				},
 			},
+			Executor: v1beta2.ExecutorSpec{
+				Lifecycle: &corev1.Lifecycle{
+					PostStart: &corev1.LifecycleHandler{Exec: postStartTest},
+				},
+			},
 		},
 	}
 
@@ -1784,11 +1792,34 @@ func TestPatchSparkPod_Lifecycle(t *testing.T) {
 		},
 	}
 
+	executorPod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "spark-executor",
+			Labels: map[string]string{
+				config.SparkRoleLabel:               config.SparkExecutorRole,
+				config.LaunchedBySparkOperatorLabel: "true",
+			},
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:  config.SparkExecutorContainerName,
+					Image: "spark-executor:latest",
+				},
+			},
+		},
+	}
+
 	modifiedDriverPod, err := getModifiedPod(driverPod, app)
 	if err != nil {
 		t.Fatal(err)
 	}
+	modifiedExecutorPod, err := getModifiedPod(executorPod, app)
+	if err != nil {
+		t.Fatal(err)
+	}
 	assert.Equal(t, preStopTest, modifiedDriverPod.Spec.Containers[0].Lifecycle.PreStop.Exec)
+	assert.Equal(t, postStartTest, modifiedExecutorPod.Spec.Containers[0].Lifecycle.PostStart.Exec)
 }
 
 func getModifiedPod(pod *corev1.Pod, app *v1beta2.SparkApplication) (*corev1.Pod, error) {