Repository: GoogleCloudPlatform/spark-on-k8s-operator
Pull request #1758 (2023-05-22 15:16:37)

Handle invalid API resources in discovery
Solution based in a similar issue fixed in helm: [helm/helm#6361](https://github.com/helm/helm/issues/6361)

To reproduce the problem, just deploy an `apiservice`

```bash
$ oc apply -f https://raw.githubusercontent.com/kubernetes/sample-apiserver/master/artifacts/example/apiservice.yaml
```

This gonna create an error in your cluster

```
$ oc api-resources > /dev/null
error: unable to retrieve the complete list of server APIs: wardle.example.com/v1alpha1: the server is currently unable to handle the request
```

After this, the Spark Operator `pod` will stop with the same error above.

This PR prevents this using the [IsGroupDiscoveryFailedError](https://pkg.go.dev/k8s.io/client-go@v1.5.2/1.5/discovery#IsGroupDiscoveryFailedError) method.

google-cla[bot]: Thanks for your pull request! It looks like this may be your first contribution to a Google open source project. Before we can look at your pull request, you'll need to sign a Contributor License Agreement (CLA).

View this [failed invocation](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/pull/1758/checks?check_run_id=13663870937) of the CLA check for more information.

For the most up to date status, view the checks section at the bottom of the pull request.
wiltonsr: Solution based in a similar issue fixed in helm: [helm/helm#6361](https://github.com/helm/helm/issues/6361)
wiltonsr: To reproduce the problem, just deploy an `apiservice`

```bash
$ oc apply -f https://raw.githubusercontent.com/kubernetes/sample-apiserver/master/artifacts/example/apiservice.yaml
```

This gonna create an error in your cluster

```
$ oc api-resources > /dev/null
error: unable to retrieve the complete list of server APIs: wardle.example.com/v1alpha1: the server is currently unable to handle the request
```

After this, the Spark Operator `pod` will stop with the same error above.

This PR prevents this using the [IsGroupDiscoveryFailedError](https://pkg.go.dev/k8s.io/client-go@v1.5.2/1.5/discovery#IsGroupDiscoveryFailedError) method.

Patch:
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
index c896e0a..9f2afd4 100644
--- a/.github/workflows/main.yaml
+++ b/.github/workflows/main.yaml
@@ -98,7 +98,7 @@ jobs:
           done
 
   build-helm-chart:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Checkout source code
         uses: actions/checkout@v2
@@ -153,7 +153,7 @@ jobs:
           ct install
 
   integration-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Checkout source code
         uses: actions/checkout@v2
diff --git a/charts/spark-operator-chart/Chart.yaml b/charts/spark-operator-chart/Chart.yaml
index da9a2e4..437804b 100644
--- a/charts/spark-operator-chart/Chart.yaml
+++ b/charts/spark-operator-chart/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: spark-operator
 description: A Helm chart for Spark on Kubernetes operator
-version: 1.1.27
+version: 1.1.28
 appVersion: v1beta2-1.3.8-3.1.1
 keywords:
   - spark
diff --git a/charts/spark-operator-chart/README.md b/charts/spark-operator-chart/README.md
index 2314c82..50c7cd8 100644
--- a/charts/spark-operator-chart/README.md
+++ b/charts/spark-operator-chart/README.md
@@ -112,6 +112,7 @@ All charts linted successfully
 | rbac.create | bool | `false` | **DEPRECATED** use `createRole` and `createClusterRole` |
 | rbac.createClusterRole | bool | `true` | Create and use RBAC `ClusterRole` resources |
 | rbac.createRole | bool | `true` | Create and use RBAC `Role` resources |
+| rbac.annotations | object | `{}` | Optional annotations for the spark rbac |
 | replicaCount | int | `1` | Desired number of pods, leaderElection will be enabled if this is greater than 1 |
 | resourceQuotaEnforcement.enable | bool | `false` | Whether to enable the ResourceQuota enforcement for SparkApplication resources. Requires the webhook to be enabled by setting `webhook.enable` to true. Ref: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/user-guide.md#enabling-resource-quota-enforcement. |
 | resources | object | `{}` | Pod resource requests and limits Note, that each job submission will spawn a JVM within the Spark Operator Pod using "/usr/local/openjdk-11/bin/java -Xmx128m". Kubernetes may kill these Java processes at will to enforce resource limits. When that happens, you will see the following error: 'failed to run spark-submit for SparkApplication [...]: signal: killed' - when this happens, you may want to increase memory limits. |
diff --git a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
index 1d23c4d..20e1e00 100644
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
@@ -919,6 +919,11 @@ spec:
                                 type: object
                               securityContext:
                                 properties:
+                                  seccompProfile:
+                                    type: object
+                                    properties:
+                                      type:
+                                        type: string
                                   allowPrivilegeEscalation:
                                     type: boolean
                                   capabilities:
@@ -1272,6 +1277,11 @@ spec:
                           type: array
                         securityContext:
                           properties:
+                            seccompProfile:
+                              type: object
+                              properties:
+                                type:
+                                  type: string
                             allowPrivilegeEscalation:
                               type: boolean
                             capabilities:
@@ -1704,6 +1714,11 @@ spec:
                                 type: object
                               securityContext:
                                 properties:
+                                  seccompProfile:
+                                    type: object
+                                    properties:
+                                      type:
+                                        type: string
                                   allowPrivilegeEscalation:
                                     type: boolean
                                   capabilities:
@@ -2731,6 +2746,11 @@ spec:
                                 type: object
                               securityContext:
                                 properties:
+                                  seccompProfile:
+                                    type: object
+                                    properties:
+                                      type:
+                                        type: string
                                   allowPrivilegeEscalation:
                                     type: boolean
                                   capabilities:
@@ -2980,6 +3000,11 @@ spec:
                           type: array
                         securityContext:
                           properties:
+                            seccompProfile:
+                              type: object
+                              properties:
+                                type:
+                                  type: string
                             allowPrivilegeEscalation:
                               type: boolean
                             capabilities:
@@ -3408,6 +3433,11 @@ spec:
                                 type: object
                               securityContext:
                                 properties:
+                                  seccompProfile:
+                                    type: object
+                                    properties:
+                                      type:
+                                        type: string
                                   allowPrivilegeEscalation:
                                     type: boolean
                                   capabilities:
diff --git a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
index 5ca3100..63e70b2 100644
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
@@ -905,6 +905,11 @@ spec:
                             type: object
                           securityContext:
                             properties:
+                              seccompProfile:
+                                type: object
+                                properties:
+                                  type:
+                                    type: string
                               allowPrivilegeEscalation:
                                 type: boolean
                               capabilities:
@@ -1258,6 +1263,11 @@ spec:
                       type: array
                     securityContext:
                       properties:
+                        seccompProfile:
+                          type: object
+                          properties:
+                            type:
+                              type: string
                         allowPrivilegeEscalation:
                           type: boolean
                         capabilities:
@@ -1690,6 +1700,11 @@ spec:
                             type: object
                           securityContext:
                             properties:
+                              seccompProfile:
+                                type: object
+                                properties:
+                                  type:
+                                    type: string
                               allowPrivilegeEscalation:
                                 type: boolean
                               capabilities:
@@ -2717,6 +2732,11 @@ spec:
                             type: object
                           securityContext:
                             properties:
+                              seccompProfile:
+                                type: object
+                                properties:
+                                  type:
+                                    type: string
                               allowPrivilegeEscalation:
                                 type: boolean
                               capabilities:
@@ -2966,6 +2986,11 @@ spec:
                       type: array
                     securityContext:
                       properties:
+                        seccompProfile:
+                          type: object
+                          properties:
+                            type:
+                              type: string
                         allowPrivilegeEscalation:
                           type: boolean
                         capabilities:
@@ -3394,6 +3419,11 @@ spec:
                             type: object
                           securityContext:
                             properties:
+                              seccompProfile:
+                                type: object
+                                properties:
+                                  type:
+                                    type: string
                               allowPrivilegeEscalation:
                                 type: boolean
                               capabilities:
diff --git a/charts/spark-operator-chart/templates/deployment.yaml b/charts/spark-operator-chart/templates/deployment.yaml
index 127a2dd..9deacae 100644
--- a/charts/spark-operator-chart/templates/deployment.yaml
+++ b/charts/spark-operator-chart/templates/deployment.yaml
@@ -47,6 +47,8 @@ spec:
       - name: {{ .Chart.Name }}
         image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
+        envFrom:
+          {{- toYaml .Values.envFrom | nindent 10 }}
         securityContext:
           {{- toYaml .Values.securityContext | nindent 10 }}
         {{- if .Values.metrics.enable }}
@@ -98,6 +100,9 @@ spec:
         {{- with .Values.volumeMounts }}
         {{- toYaml . | nindent 10 }}
         {{- end }}
+      {{- with .Values.sidecars }}
+        {{- toYaml . | nindent 6 }}
+      {{- end }}
       {{- if or .Values.webhook.enable (ne (len .Values.volumes) 0 ) }}
       volumes:
       {{- end }}
diff --git a/charts/spark-operator-chart/templates/rbac.yaml b/charts/spark-operator-chart/templates/rbac.yaml
index 6f5d97c..c78a73d 100644
--- a/charts/spark-operator-chart/templates/rbac.yaml
+++ b/charts/spark-operator-chart/templates/rbac.yaml
@@ -7,6 +7,9 @@ metadata:
     "helm.sh/hook": pre-install, pre-upgrade
     "helm.sh/hook-delete-policy": hook-failed, before-hook-creation
     "helm.sh/hook-weight": "-10"
+{{- with .Values.rbac.annotations }}
+{{ toYaml . | indent 4 }}
+{{- end }}
   labels:
     {{- include "spark-operator.labels" . | nindent 4 }}
 rules:
diff --git a/charts/spark-operator-chart/values.yaml b/charts/spark-operator-chart/values.yaml
index 5007b7e..3436a8f 100644
--- a/charts/spark-operator-chart/values.yaml
+++ b/charts/spark-operator-chart/values.yaml
@@ -33,6 +33,8 @@ rbac:
   createRole: true
   # -- Create and use RBAC `ClusterRole` resources
   createClusterRole: true
+  # -- Optional annotations for rbac
+  annotations: {}
 
 serviceAccounts:
   spark:
@@ -71,12 +73,18 @@ ingressUrlFormat: ""
 # -- Set higher levels for more verbose logging
 logLevel: 2
 
+# -- Pod environment variable sources
+envFrom: []
+
 # podSecurityContext -- Pod security context
 podSecurityContext: {}
 
 # securityContext -- Operator container security context
 securityContext: {}
 
+# sidecars -- Sidecar containers
+sidecars: []
+
 # volumes - Operator volumes
 volumes: []
 
diff --git a/docs/api-docs.md b/docs/api-docs.md
index 5889e41..0403b6e 100644
--- a/docs/api-docs.md
+++ b/docs/api-docs.md
@@ -1293,6 +1293,20 @@ GC settings or other logging.</p>
 </tr>
 <tr>
 <td>
+<code>lifecycle</code><br/>
+<em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#lifecycle-v1-core">
+Kubernetes core/v1.Lifecycle
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Lifecycle for running preStop or postStart commands</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>deleteOnTermination</code><br/>
 <em>
 bool
diff --git a/docs/quick-start-guide.md b/docs/quick-start-guide.md
index 22e8cbb..01dd103 100644
--- a/docs/quick-start-guide.md
+++ b/docs/quick-start-guide.md
@@ -18,7 +18,7 @@ For a more detailed guide on how to use, compose, and work with `SparkApplicatio
       - [Work Queue Metrics](#work-queue-metrics)
   - [Driver UI Access and Ingress](#driver-ui-access-and-ingress)
   - [About the Mutating Admission Webhook](#about-the-mutating-admission-webhook)
-    - [Mutating Admission Webhooks on a private GKE cluster](#mutating-admission-webhooks-on-a-private-gke-cluster)
+    - [Mutating Admission Webhooks on a private GKE or EKS cluster](#mutating-admission-webhooks-on-a-private-gke-or-eks-cluster)
 
 ## Installation
 
@@ -162,7 +162,7 @@ To upgrade the the operator, e.g., to use a newer version container image with a
 $ helm upgrade <YOUR-HELM-RELEASE-NAME> --set image.repository=org/image --set image.tag=newTag
 ```
 
-Refer to the Helm [documentation](https://docs.helm.sh/helm/#helm-upgrade) for more details on `helm upgrade`.
+Refer to the Helm [documentation](https://helm.sh/docs/helm/helm_upgrade/) for more details on `helm upgrade`.
 
 ## About the Spark Job Namespace
 
@@ -238,10 +238,20 @@ and deleting the pods outside the operator might lead to incorrect metric values
 ## Driver UI Access and Ingress
 
 The operator, by default, makes the Spark UI accessible by creating a service of type `ClusterIP` which exposes the UI. This is only accessible from within the cluster.
+
 The operator also supports creating an optional Ingress for the UI. This can be turned on by setting the `ingress-url-format` command-line flag. The `ingress-url-format` should be a template like `{{$appName}}.{ingress_suffix}/{{$appNamespace}}/{{$appName}}`. The `{ingress_suffix}` should be replaced by the user to indicate the cluster's ingress url and the operator will replace the `{{$appName}}` & `{{$appNamespace}}` with the appropriate value. Please note that Ingress support requires that cluster's ingress url routing is correctly set-up. For e.g. if the `ingress-url-format` is `{{$appName}}.ingress.cluster.com`, it requires that anything `*ingress.cluster.com` should be routed to the ingress-controller on the K8s cluster.
 
 The operator also sets both `WebUIAddress` which is accessible from within the cluster as well as `WebUIIngressAddress` as part of the `DriverInfo` field of the `SparkApplication`.
 
+The operator generates ingress resources intended for use with the [Ingress NGINX Controller](https://kubernetes.github.io/ingress-nginx/). Include this in your application spec for the controller to ensure it recognizes the ingress and provides appropriate routes to your Spark UI.
+
+```yaml
+spec:
+  sparkUIOptions:
+    ingressAnnotations:
+        kubernetes.io/ingress.class: nginx
+```
+
 ## About the Mutating Admission Webhook
 
 The Kubernetes Operator for Apache Spark comes with an optional mutating admission webhook for customizing Spark driver and executor pods based on the specification in `SparkApplication` objects, e.g., mounting user-specified ConfigMaps and volumes, and setting pod affinity/anti-affinity, and adding tolerations.
@@ -257,9 +267,9 @@ $ kubectl apply -f manifest/spark-operator-with-webhook.yaml
 
 This will create a Deployment named `sparkoperator` and a Service named `spark-webhook` for the webhook in namespace `spark-operator`.
 
-### Mutating Admission Webhooks on a private GKE cluster
+### Mutating Admission Webhooks on a private GKE or EKS cluster
 
-If you are deploying the operator on a GKE cluster with the [Private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters) setting enabled, and you wish to deploy the cluster with the [Mutating Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), then make sure to change the `webhookPort` to `443`. Alternatively you can choose to allow connections to the default port (8080).
+If you are deploying the operator on a GKE cluster with the [Private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters) setting enabled, or on an enterprise AWS EKS cluster and you wish to deploy the cluster with the [Mutating Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), then make sure to change the `webhookPort` to `443`. Alternatively you can choose to allow connections to the default port (8080).
 
 > By default, firewall rules restrict your cluster master to only initiate TCP connections to your nodes on ports 443 (HTTPS) and 10250 (kubelet). For some Kubernetes features, you might need to add firewall rules to allow access on additional ports. For example, in Kubernetes 1.9 and older, kubectl top accesses heapster, which needs a firewall rule to allow TCP connections on port 8080. To grant such access, you can add firewall rules.
 [From the docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules)
diff --git a/docs/volcano-integration.md b/docs/volcano-integration.md
index 30cd91f..ecb325d 100644
--- a/docs/volcano-integration.md
+++ b/docs/volcano-integration.md
@@ -16,7 +16,7 @@ same environment, please refer [Quick Start Guide](https://github.com/volcano-sh
 Within the help of Helm chart, Kubernetes Operator for Apache Spark with Volcano can be easily installed with the command below:
 ```bash
 $ helm repo add spark-operator https://googlecloudplatform.github.io/spark-on-k8s-operator
-$ helm install my-release spark-operator/spark-operator --namespace spark-operator --set enableBatchScheduler=true --set enableWebhook=true
+$ helm install my-release spark-operator/spark-operator --namespace spark-operator --set batchScheduler.enable=true --set webhook.enable=true
 ```
 
 # Run Spark Application with Volcano scheduler
diff --git a/docs/who-is-using.md b/docs/who-is-using.md
index f0c5759..4f5e41f 100644
--- a/docs/who-is-using.md
+++ b/docs/who-is-using.md
@@ -41,3 +41,5 @@
 | [C2FO](https://www.c2fo.com/) | @vanhoale | Production | Data Platform / Data Infrastructure |
 | [Kognita](https://kognita.com.br/) | @andreclaudino | Production | MLOps, Data Platform / Data Infrastructure, ML/AI |
 | [Molex](https://www.molex.com/) | @AshishPushpSingh | Evaluation/Production | Data Platform |
+| [Qualytics](https://www.qualytics.co/) | @josecsotomorales | Production | Data Quality Platform |
+| [Roblox](https://www.roblox.com/) | @matschaffer-roblox | Evaluation | Data Infrastructure |
diff --git a/manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml b/manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
index 3ee45fb..20e1e00 100644
--- a/manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+++ b/manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
@@ -919,6 +919,11 @@ spec:
                                 type: object
                               securityContext:
                                 properties:
+                                  seccompProfile:
+                                    type: object
+                                    properties:
+                                      type:
+                                        type: string
                                   allowPrivilegeEscalation:
                                     type: boolean
                                   capabilities:
@@ -1272,6 +1277,11 @@ spec:
                           type: array
                         securityContext:
                           properties:
+                            seccompProfile:
+                              type: object
+                              properties:
+                                type:
+                                  type: string
                             allowPrivilegeEscalation:
                               type: boolean
                             capabilities:
@@ -1704,6 +1714,11 @@ spec:
                                 type: object
                               securityContext:
                                 properties:
+                                  seccompProfile:
+                                    type: object
+                                    properties:
+                                      type:
+                                        type: string
                                   allowPrivilegeEscalation:
                                     type: boolean
                                   capabilities:
@@ -2731,6 +2746,11 @@ spec:
                                 type: object
                               securityContext:
                                 properties:
+                                  seccompProfile:
+                                    type: object
+                                    properties:
+                                      type:
+                                        type: string
                                   allowPrivilegeEscalation:
                                     type: boolean
                                   capabilities:
@@ -2980,6 +3000,11 @@ spec:
                           type: array
                         securityContext:
                           properties:
+                            seccompProfile:
+                              type: object
+                              properties:
+                                type:
+                                  type: string
                             allowPrivilegeEscalation:
                               type: boolean
                             capabilities:
@@ -3408,6 +3433,11 @@ spec:
                                 type: object
                               securityContext:
                                 properties:
+                                  seccompProfile:
+                                    type: object
+                                    properties:
+                                      type:
+                                        type: string
                                   allowPrivilegeEscalation:
                                     type: boolean
                                   capabilities:
@@ -3936,6 +3966,29 @@ spec:
                                 pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                                 x-kubernetes-int-or-string: true
                             type: object
+                          ephemeral:
+                            properties:
+                              volumeClaimTemplate:
+                                properties:
+                                  spec:
+                                    properties:
+                                      accessModes:
+                                        items:
+                                          type: string
+                                        type: array
+                                      resources:
+                                        properties:
+                                          requests:
+                                            properties:
+                                              storage:
+                                                type: string
+                                            type: object
+                                        type: object
+                                      storageClassName:
+                                        type: string
+                                    type: object
+                                type: object
+                            type: object
                           fc:
                             properties:
                               fsType:
diff --git a/manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml b/manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml
index fbb52e2..63e70b2 100644
--- a/manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml
+++ b/manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml
@@ -905,6 +905,11 @@ spec:
                             type: object
                           securityContext:
                             properties:
+                              seccompProfile:
+                                type: object
+                                properties:
+                                  type:
+                                    type: string
                               allowPrivilegeEscalation:
                                 type: boolean
                               capabilities:
@@ -1258,6 +1263,11 @@ spec:
                       type: array
                     securityContext:
                       properties:
+                        seccompProfile:
+                          type: object
+                          properties:
+                            type:
+                              type: string
                         allowPrivilegeEscalation:
                           type: boolean
                         capabilities:
@@ -1690,6 +1700,11 @@ spec:
                             type: object
                           securityContext:
                             properties:
+                              seccompProfile:
+                                type: object
+                                properties:
+                                  type:
+                                    type: string
                               allowPrivilegeEscalation:
                                 type: boolean
                               capabilities:
@@ -2717,6 +2732,11 @@ spec:
                             type: object
                           securityContext:
                             properties:
+                              seccompProfile:
+                                type: object
+                                properties:
+                                  type:
+                                    type: string
                               allowPrivilegeEscalation:
                                 type: boolean
                               capabilities:
@@ -2966,6 +2986,11 @@ spec:
                       type: array
                     securityContext:
                       properties:
+                        seccompProfile:
+                          type: object
+                          properties:
+                            type:
+                              type: string
                         allowPrivilegeEscalation:
                           type: boolean
                         capabilities:
@@ -3394,6 +3419,11 @@ spec:
                             type: object
                           securityContext:
                             properties:
+                              seccompProfile:
+                                type: object
+                                properties:
+                                  type:
+                                    type: string
                               allowPrivilegeEscalation:
                                 type: boolean
                               capabilities:
@@ -3924,6 +3954,29 @@ spec:
                             pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                             x-kubernetes-int-or-string: true
                         type: object
+                      ephemeral:
+                        properties:
+                          volumeClaimTemplate:
+                            properties:
+                              spec:
+                                properties:
+                                  accessModes:
+                                    items:
+                                      type: string
+                                    type: array
+                                  resources:
+                                    properties:
+                                      requests:
+                                        properties:
+                                          storage:
+                                            type: string
+                                        type: object
+                                    type: object
+                                  storageClassName:
+                                    type: string
+                                type: object
+                            type: object
+                        type: object
                       fc:
                         properties:
                           fsType:
diff --git a/pkg/controller/sparkapplication/submission.go b/pkg/controller/sparkapplication/submission.go
index b8e4e1b..98bb882 100644
--- a/pkg/controller/sparkapplication/submission.go
+++ b/pkg/controller/sparkapplication/submission.go
@@ -207,6 +207,10 @@ func getMasterURL() (string, error) {
 	if kubernetesServicePort == "" {
 		return "", fmt.Errorf("environment variable %s is not found", kubernetesServicePortEnvVar)
 	}
+	// check if the host is IPv6 address
+	if strings.Contains(kubernetesServiceHost, ":") && !strings.HasPrefix(kubernetesServiceHost, "[") {
+		return fmt.Sprintf("k8s://https://[%s]:%s", kubernetesServiceHost, kubernetesServicePort), nil
+	}
 	return fmt.Sprintf("k8s://https://%s:%s", kubernetesServiceHost, kubernetesServicePort), nil
 }
 
diff --git a/pkg/util/capabilities.go b/pkg/util/capabilities.go
index 27b7f71..5040da6 100644
--- a/pkg/util/capabilities.go
+++ b/pkg/util/capabilities.go
@@ -19,6 +19,8 @@ package util
 import (
 	"strings"
 
+	"github.com/golang/glog"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -44,7 +46,11 @@ func getPreferredAvailableAPIs(client kubernetes.Interface, kind string) (Capabi
 	discoveryclient := client.Discovery()
 	lists, err := discoveryclient.ServerPreferredResources()
 	if err != nil {
-		return nil, err
+		if discovery.IsGroupDiscoveryFailedError(err) {
+			glog.Infof("There is an orphaned API service. Server reports: %s", err)
+		} else {
+			return nil, err
+		}
 	}
 
 	caps := Capabilities{}


Test patch:
diff --git a/pkg/controller/sparkapplication/submission_test.go b/pkg/controller/sparkapplication/submission_test.go
index 9f32bbb..20e247a 100644
--- a/pkg/controller/sparkapplication/submission_test.go
+++ b/pkg/controller/sparkapplication/submission_test.go
@@ -583,3 +583,54 @@ func TestProxyUserArg(t *testing.T) {
 	assert.Equal(t, "--proxy-user", args[4])
 	assert.Equal(t, "foo", args[5])
 }
+
+func Test_getMasterURL(t *testing.T) {
+	setEnv := func(host string, port string) {
+		if err := os.Setenv(kubernetesServiceHostEnvVar, host); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.Setenv(kubernetesServicePortEnvVar, port); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	tests := []struct {
+		name    string
+		host    string
+		port    string
+		want    string
+		wantErr assert.ErrorAssertionFunc
+	}{
+		{
+			name:    "should return a valid master url when IPv4 address is used",
+			host:    "localhost",
+			port:    "6443",
+			want:    "k8s://https://localhost:6443",
+			wantErr: assert.NoError,
+		},
+		{
+			name:    "should return a valid master url when IPv6 address is used",
+			host:    "::1",
+			port:    "6443",
+			want:    "k8s://https://[::1]:6443",
+			wantErr: assert.NoError,
+		},
+		{
+			name:    "should throw an error when the host is empty",
+			host:    "",
+			port:    "6443",
+			want:    "",
+			wantErr: assert.Error,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setEnv(tt.host, tt.port)
+			got, err := getMasterURL()
+			if !tt.wantErr(t, err, fmt.Sprintf("getMasterURL()")) {
+				return
+			}
+			assert.Equalf(t, tt.want, got, "getMasterURL()")
+		})
+	}
+}