Change:
In we observed a crashlooping controller cause a cluster cold boot failure, as controller-manager wasn't able to stay up long enough to run the deployment that would have fixed the problem. We should audit everything that runs in kube-controller-manager and ensure that they all fail gracefully enough to prevent crashloops, or make them individually restart in that case, instead of the process. Longer term, we should consider classifying the controllers by run level or bootstrap order (), and splitting them into separate binaries or changing the start-up order in kcm to make sure this sort of thing doesn't happen in the future.
Same issue with metrics-server, using minikube. Also reproduced on our test cluster. Steps to reproduce: Set up localkube Install metrics-server Stop minikube: sudo minikube stop Start minikube: sudo -E minikube start --vm-driver=none Expected: Kubernetes is running Actual: Kubernetes is dead Logs: Workaround: To fix the system, we must delete the metrics-server: Now kubernetes will start normally. After that we can reinstall the metrics server, and it will be ok.
Related:
This will be a hard problem to fix permanently. In all likelihood, we should stop allowing Fatalf() in kubernetes/kubernetes, but as you can see we do this to ourselves right here...
/kind bug
I also think this bug would benefit from an e2e test where we misconfigure things in a variety of ways and verify that KCM still comes up and stays up: [ ] misconfigured apiservice [ ] broken service attached to that apiservice [ ] RBAC rules that disallow things. maybe everything! [ ] webhooks that disallow everything [ ] yr example here
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale

Discussion:
cacheddiscovery "k8s.io/client-go/discovery/cached" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" <ins> restclient "k8s.io/client-go/rest" </ins> csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" "k8s.io/kubernetes/pkg/controller" cloudcontroller "k8s.io/kubernetes/pkg/controller/cloud"

nsKubeconfig.QPS *= 20 nsKubeconfig.Burst *= 100 namespaceKubeClient := clientset.NewForConfigOrDie(nsKubeconfig) <ins> return startModifiedNamespaceController(ctx, namespaceKubeClient, nsKubeconfig) } func startModifiedNamespaceController(ctx ControllerContext, namespaceKubeClient clientset.Interface, nsKubeconfig *restclient.Config) (http.Handler, bool, error) { </ins> dynamicClient, err := dynamic.NewForConfig(nsKubeconfig) if err != nil {

package app import ( <ins> "net/http" </ins> "testing" "time"

clientset clientset.Interface } <del> func (TestClientBuilder) Config(name string) (*restclient.Config, error)  { return nil, nil } func (TestClientBuilder) ConfigOrDie(name string) *restclient.Config      { return nil } </del> <ins> func (TestClientBuilder) Config(name string) (*restclient.Config, error) { return nil, nil } func (TestClientBuilder) ConfigOrDie(name string) *restclient.Config { return &restclient.Config{} } </ins> func (TestClientBuilder) Client(name string) (clientset.Interface, error) { return nil, nil } func (m TestClientBuilder) ClientOrDie(name string) clientset.Interface { return m.clientset

return c.DiscoveryObj } <ins> func (c *FakeClientSet) GetPossibleResources() []*metav1.APIResourceList { return c.DiscoveryObj.PossibleResources } </ins> // Create a fake Clientset with its Discovery method overridden. func NewFakeClientset(fakeDiscovery FakeDiscoveryWithError) *FakeClientSet { cs := &FakeClientSet{}

return cs } <del> func TestStartResourceQuotaController_DiscoveryError(t *testing.T) { </del> <ins> func possibleDiscoveryResource() []*metav1.APIResourceList { return []*metav1.APIResourceList{ { GroupVersion: "create/v1", APIResources: []metav1.APIResource{ { Name:       "jobs", Verbs:      []string{"create", "list", "watch", "delete"}, ShortNames: []string{"jz"}, Categories: []string{"all"}, }, }, }, } } type controllerInitFunc func(ControllerContext) (http.Handler, bool, error) func TestController_DiscoveryError(t *testing.T) { controllerInitFuncMap := map[string]controllerInitFunc{ "ResourceQuotaController":    startResourceQuotaController, "GarbageCollectorController": startGarbageCollectorController, } </ins> tcs := map[string]struct { discoveryError    error expectedErr       bool

}{ "No Discovery Error": { discoveryError:    nil, <del> possibleResources: nil, </del> <ins> possibleResources: possibleDiscoveryResource(), </ins> expectedErr:       false, }, "Discovery Calls Partially Failed": { <del> discoveryError: new(discovery.ErrGroupDiscoveryFailed), possibleResources: []*metav1.APIResourceList{ { GroupVersion: "create/v1", APIResources: []metav1.APIResource{ { Name:       "jobs", Verbs:      []string{"create", "list", "watch", "delete"}, ShortNames: []string{"jz"}, Categories: []string{"all"}, }, }, }, }, expectedErr: false, </del> <ins> discoveryError:    new(discovery.ErrGroupDiscoveryFailed), possibleResources: possibleDiscoveryResource(), expectedErr:       false, </ins> }, } for name, test := range tcs {

InformerFactory:  informers.NewSharedInformerFactoryWithOptions(testClientset, time.Duration(1)), InformersStarted: make(chan struct{}), } <del> _, _, err := startResourceQuotaController(ctx) </del> <ins> for funcName, controllerInit := range controllerInitFuncMap { _, _, err := controllerInit(ctx) if test.expectedErr != (err != nil) { t.Errorf("%v test failed for use case: %v", funcName, name) } } _, _, err := startModifiedNamespaceController( ctx, testClientset, testClientBuilder.ConfigOrDie("namespace-controller")) </ins> if test.expectedErr != (err != nil) { <del> t.Errorf("test failed for use case: %v", name) </del> <ins> t.Errorf("Namespace Controller test failed for use case: %v", name) </ins> } } }