Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: kubectl drain crashed with panic when apiserver connection disappeared while there were pods that couldn't be evicted/deleted (sorry, the problem is a bit hard to reproduce and the log was lost) What you expected to happen: kubectl drain should exit with a proper error message How to reproduce it (as minimally and precisely as possible): Try kubectl drain multiple times, interrupting the connection during eviction/deletion Anything else we need to know?: The here can be in case if returns an error, and will panic upon accessing : This causes the c Environment: Kubernetes version (use ): The problem is present as of but is also present in earlier versions (I have a fix for this bug)
/priority important-longterm /sig cli

Discussion:
if err := o.drainer.DeleteOrEvictPods(list.Pods()); err != nil { pendingList, newErrs := o.drainer.GetPodsForDeletion(nodeInfo.Name) <del> fmt.Fprintf(o.ErrOut, "There are pending pods in node %q when an error occurred: %vn", nodeInfo.Name, err) for _, pendingPod := range pendingList.Pods() { fmt.Fprintf(o.ErrOut, "%s/%sn", "pod", pendingPod.Name) </del> <ins> if pendingList != nil { pods := pendingList.Pods() if len(pods) != 0 { fmt.Fprintf(o.ErrOut, "There are pending pods in node %q when an error occurred: %vn", nodeInfo.Name, err) for _, pendingPod := range pods { fmt.Fprintf(o.ErrOut, "%s/%sn", "pod", pendingPod.Name) } } </ins> } if newErrs != nil { <del> fmt.Fprintf(o.ErrOut, "following errors also occurred:n%s", utilerrors.NewAggregate(newErrs)) </del> <ins> fmt.Fprintf(o.ErrOut, "Following errors occurred while getting the list of pods to delete:n%s", utilerrors.NewAggregate(newErrs)) </ins> } return err }

package drain import ( <ins> "errors" </ins> "io/ioutil" "net/http" "net/url"

ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() cmd := test.cmd(tf, ioStreams) <ins> var recovered interface{} </ins> sawFatal := false func() { defer func() { // Recover from the panic below. <del> _ = recover() </del> <ins> recovered = recover() </ins> // Restore cmdutil behavior cmdutil.DefaultBehaviorOnFatal() }()

cmd.Execute() }() <del> if test.expectFatal { </del> <ins> switch { case recovered != nil && !sawFatal: t.Fatalf("got panic: %v", recovered) case test.expectFatal: </ins> if !sawFatal { t.Fatalf("%s: unexpected non-error", test.description) } if updated { t.Fatalf("%s: unexpected update", test.description) } <del> } if !test.expectFatal && sawFatal { </del> <ins> case !test.expectFatal && sawFatal: </ins> t.Fatalf("%s: unexpected error", test.description) <del> } if !reflect.DeepEqual(test.expected.Spec, test.node.Spec) && !updated { </del> <ins> case !reflect.DeepEqual(test.expected.Spec, test.node.Spec) && !updated: </ins> t.Fatalf("%s: node never updated", test.description) } })

} tests := []struct { <del> description   string node          *corev1.Node expected      *corev1.Node pods          []corev1.Pod rcs           []corev1.ReplicationController replicaSets   []appsv1.ReplicaSet args          []string expectWarning string expectFatal   bool expectDelete  bool </del> <ins> description                string node                       *corev1.Node expected                   *corev1.Node pods                       []corev1.Pod rcs                        []corev1.ReplicationController replicaSets                []appsv1.ReplicaSet args                       []string failUponEvictionOrDeletion bool expectWarning              string expectFatal                bool expectDelete               bool </ins> }{ { description:  "RC-managed pod",

expectFatal:  false, expectDelete: false, }, <ins> { description:                "fail to list pods", node:                       node, expected:                   cordonedNode, pods:                       []corev1.Pod{rsPod}, replicaSets:                []appsv1.ReplicaSet{rs}, args:                       []string{"node"}, expectFatal:                true, expectDelete:               true, failUponEvictionOrDeletion: true, }, </ins> } testEviction := false

case m.isFor("GET", "/namespaces/default/pods/bar"): return &http.Response{StatusCode: 404, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, &corev1.Pod{})}, nil case m.isFor("GET", "/pods"): <ins> if test.failUponEvictionOrDeletion && atomic.LoadInt32(&evictions) > 0 || atomic.LoadInt32(&deletions) > 0 { return nil, errors.New("request failed") } </ins> values, err := url.ParseQuery(req.URL.RawQuery) if err != nil { t.Fatalf("%s: unexpected error: %v", test.description, err)

return &http.Response{StatusCode: 200, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, newNode)}, nil case m.isFor("DELETE", "/namespaces/default/pods/bar"): atomic.AddInt32(&deletions, 1) <ins> if test.failUponEvictionOrDeletion { return nil, errors.New("request failed") } </ins> return &http.Response{StatusCode: 204, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, &test.pods[0])}, nil case m.isFor("POST", "/namespaces/default/pods/bar/eviction"): atomic.AddInt32(&evictions, 1) <ins> if test.failUponEvictionOrDeletion { return nil, errors.New("request failed") } </ins> return &http.Response{StatusCode: 201, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, &policyv1beta1.Eviction{})}, nil default: t.Fatalf("%s: unexpected request: %v %#vn%#v", test.description, req.Method, req.URL, req)

ioStreams, _, _, errBuf := genericclioptions.NewTestIOStreams() cmd := NewCmdDrain(tf, ioStreams) <ins> var recovered interface{} </ins> sawFatal := false fatalMsg := "" func() { defer func() { // Recover from the panic below. <del> _ = recover() </del> <ins> recovered = recover() </ins> // Restore cmdutil behavior cmdutil.DefaultBehaviorOnFatal() }()

cmd.SetArgs(test.args) cmd.Execute() }() <del> if test.expectFatal { if !sawFatal { //t.Logf("outBuf = %s", outBuf.String()) //t.Logf("errBuf = %s", errBuf.String()) t.Fatalf("%s: unexpected non-error when using %s", test.description, currMethod) } } else { if sawFatal { t.Fatalf("%s: unexpected error when using %s: %s", test.description, currMethod, fatalMsg) } </del> <ins> switch { case recovered != nil && !sawFatal: t.Fatalf("got panic: %v", recovered) case test.expectFatal && !sawFatal: t.Fatalf("%s: unexpected non-error when using %s", test.description, currMethod) case !test.expectFatal && sawFatal: t.Fatalf("%s: unexpected error when using %s: %s", test.description, currMethod, fatalMsg) </ins> } deleted := deletions > 0