Change:
What would you like to be : Add support to the kube-apiserver so that we can disable health-checks via query params. Why is this needed: Currently, the healthz endpoint is inflexible in that any and all health checks are currently executed when the endpoint is hit. Since the kube-apiserver only currently supports a liveness check, this means all persistent health check failures will eventually result in a kubelet driven restart. However, this may not be desirable behavior if you have additional monitoring services layered on top of your control plane. For instance, take the etcd failure from the perspective of the kube-apiserver. Currently, persistent failures during the etcd health check from the api-server healthz endpoint will cause the kube-apiserver to be restarted. This sometimes helps (in the case of a faulty etcd-client connection) but also can be detrimental (the kube-apiserver restarts until the etcd cluster is functioning, then the kube-apiserver bombards the etcd cluster with requests to update local caches, saturating the etcd cluster with requests, causing it to timeout on its own liveness probes and trigger a kubelet-driven restart). Let's say that we could now disable etcd from our liveness health check. Then if we had an external monitoring service, we could ask independently ask etcd if it was healthy and we could also ask the kube-apiserver if it thought etcd was unhealthy. Based on the combination of those answers, we could enable more intelligent responses to certain situation, like restarting the kube-apiserver when it thought etcd was unhealthy but etcd was actually reporting as healthy but not doing so when etcd was actually unhealthy. This feature is backwards-compatible and opt-in. There would be no impact to existing behavior unless these query params are actually used. /sig api-machinery /kind feature
/cc
SGTM
/assign

Discussion:
"httpGet": { "host": "127.0.0.1", "port": 8080, <del> "path": "/healthz" </del> <ins> "path": "/healthz?exclude=etcd" </ins> }, "initialDelaySeconds": {{liveness_probe_initial_delay}}, "timeoutSeconds": 15 }, <ins> "readinessProbe": { "httpGet": { "host": "127.0.0.1", "port": 8080, "path": "/healthz" }, "timeoutSeconds": 15 }, </ins> "ports":[ { "name": "https", "containerPort": {{secure_port}},

name = "go_default_test", srcs = ["healthz_test.go"], embed = [":go_default_library"], <ins> deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", ], </ins> ) go_library(

importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/healthz", importpath = "k8s.io/apiserver/pkg/server/healthz", deps = [ <ins> "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", </ins> "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/klog:go_default_library", ],

"k8s.io/klog" <ins> "k8s.io/apimachinery/pkg/util/sets" </ins> "k8s.io/apimachinery/pkg/util/wait" )

return c.check(r) } <ins> // getExcludedChecks extracts the health check names to be excluded from the query param func getExcludedChecks(r *http.Request) sets.String { checks, found := r.URL.Query()["exclude"] if found { return sets.NewString(checks...) } return sets.NewString() } </ins> // handleRootHealthz returns an http.HandlerFunc that serves the provided checks. func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { failed := false <ins> excluded := getExcludedChecks(r) </ins> var verboseOut bytes.Buffer for _, check := range checks { <ins> // no-op the check if we've specified we want to exclude the check if excluded.Has(check.Name()) { excluded.Delete(check.Name()) fmt.Fprintf(&verboseOut, "[+]%v excluded: okn", check.Name()) continue } </ins> if err := check.Check(r); err != nil { // don't include the error since this endpoint is public.  If someone wants more detail // they should have explicit permission to the detailed checks.

fmt.Fprintf(&verboseOut, "[+]%v okn", check.Name()) } } <ins> if excluded.Len() > 0 { fmt.Fprintf(&verboseOut, "warn: some health checks cannot be excluded: no matches for %vn", formatQuoted(excluded.List()...)) klog.Warningf("cannot exclude some health checks, no health checks are installed matching %v", formatQuoted(excluded.List()...)) } </ins> // always be verbose on failure if failed { http.Error(w, fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError)

"fmt" "net/http" "net/http/httptest" <ins> "net/url" </ins> "reflect" "testing" <ins> "k8s.io/apimachinery/pkg/util/sets" </ins> ) func TestInstallHandler(t *testing.T) {

addBadCheck      bool }{ {"?verbose", "[+]ping oknhealthz check passedn", http.StatusOK, false}, <ins> {"?exclude=dontexist", "ok", http.StatusOK, false}, {"?exclude=bad", "ok", http.StatusOK, true}, {"?verbose=true&exclude=bad", "[+]ping okn[+]bad excluded: oknhealthz check passedn", http.StatusOK, true}, {"?verbose=true&exclude=dontexist", "[+]ping oknwarn: some health checks cannot be excluded: no matches for "dontexist"nhealthz check passedn", http.StatusOK, false}, </ins> {"/ping", "ok", http.StatusOK, false}, {"", "ok", http.StatusOK, false}, {"?verbose", "[+]ping okn[-]bad failed: reason withheldnhealthz check failedn", http.StatusInternalServerError, true},

}) } } <ins> func TestGetExcludedChecks(t *testing.T) { type args struct { r *http.Request } tests := []struct { name string r    *http.Request want sets.String }{ {"Should have no excluded health checks", createGetRequestWithUrl("/healthz?verbose=true"), sets.NewString(), }, {"Should extract out the ping health check", createGetRequestWithUrl("/healthz?exclude=ping"), sets.NewString("ping"), }, {"Should extract out ping and log health check", createGetRequestWithUrl("/healthz?exclude=ping&exclude=log"), sets.NewString("ping", "log"), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := getExcludedChecks(tt.r); !reflect.DeepEqual(got, tt.want) { t.Errorf("getExcludedChecks() = %v, want %v", got, tt.want) } }) } } func createGetRequestWithUrl(rawUrlString string) *http.Request { url, _ := url.Parse(rawUrlString) return &http.Request{ Method: http.MethodGet, Proto:  "HTTP/1.1", URL:    url, } } </ins>