Change:
In writing some documentation for I noticed some issues with the implementation. I think the simplest user experience is that the command doesn't return until all the user pods are drained off the machine. In particular this means wait until all graceful termination is finished infinitely retry evictions that are rejected due to insufficient PodDisruptionBudget The current implementation does not do this, in particular wait for graceful termination appears to be at 10 sec number of pods, which may be much shorter than the max graceful termination we only retry evictions number of times My suggestion is to change the behavior so that we compute how long to wait for graceful termination, by taking max(grace periods of all pods that we want to delete) (or the value of --grace-period command line argument instead, if that is specified), and wait exactly that amount of time plus a small buffer we retry evictions infinitely WDYT? I think we should do this in 1.5 because the current behavior is very non-obvious.
The problem here is how long should we wait, if the user doesn't specify a timeout. The default timeout used in drain is following how long the reapers have in , , and . Maybe this is not long enough. I can make the changes. Do we still need exponential back off?
To summarize discussion with there are two possible approaches (pseudocode below) 1) Goroutine approach: For each pod P that meets the , do the following: 2) Non-goroutine approach: where "not timed out yet" means you haven't finished waiting max(--grace period argument to kubectl drain, max graceful termination spec across all pods on the node)
Could you please triage this for 1.5 and apply blocker or non-blocker labels as appropriate along with priority.
The PR for this issue is waiting to be merged.
marking this as a release blocker due to being marked as such. Please update if needed cc:

Discussion:
cmd.Flags().BoolVar(&options.IgnoreDaemonsets, "ignore-daemonsets", false, "Ignore DaemonSet-managed pods.") cmd.Flags().BoolVar(&options.DeleteLocalData, "delete-local-data", false, "Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).") cmd.Flags().IntVar(&options.GracePeriodSeconds, "grace-period", -1, "Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used.") <del> cmd.Flags().DurationVar(&options.Timeout, "timeout", 0, "The length of time to wait before giving up on a delete, zero means determine a timeout from the size of the object") </del> <ins> cmd.Flags().DurationVar(&options.Timeout, "timeout", 0, "The length of time to wait before giving up, zero means infinite") </ins> return cmd }

} err := o.deleteOrEvictPodsSimple() <del> // TODO: update IsTooManyRequests() when the TooManyRequests(429) error returned from the API server has a non-empty Reason field for i := 1; i <= maxPatchRetry && apierrors.IsTooManyRequests(err); i++ { if i > triesBeforeBackOff { currBackOffPeriod := time.Duration(math.Exp2(float64(i-triesBeforeBackOff))) * backOffPeriod fmt.Fprintf(o.errOut, "Retry in %vn", currBackOffPeriod) o.backOff.Sleep(currBackOffPeriod) } fmt.Fprintf(o.errOut, "Retryingn") err = o.deleteOrEvictPodsSimple() } </del> if err == nil { cmdutil.PrintSuccess(o.mapper, false, o.out, "node", o.nodeInfo.Name, false, "drained") }

if err != nil { return err } <del> if o.Timeout == 0 { o.Timeout = kubectl.Timeout + time.Duration(10*len(pods))*time.Second } </del> err = o.deleteOrEvictPods(pods) if err != nil { pendingPods, newErr := o.getPodsForDeletion()

return err } <ins> getPodFn := func(namespace, name string) (*api.Pod, error) { return o.client.Core().Pods(namespace).Get(name) } if len(policyGroupVersion) > 0 { return o.evictPods(pods, policyGroupVersion, getPodFn) } else { return o.deletePods(pods, getPodFn) } } func (o *DrainOptions) evictPods(pods []api.Pod, policyGroupVersion string, getPodFn func(namespace, name string) (*api.Pod, error)) error { doneCh := make(chan bool, len(pods)) errCh := make(chan error, 1) </ins> for _, pod := range pods { <del> if len(policyGroupVersion) > 0 { err = o.evictPod(pod, policyGroupVersion) } else { err = o.deletePod(pod) } if err != nil { </del> <ins> go func(pod api.Pod, doneCh chan bool, errCh chan error) { var err error for { err = o.evictPod(pod, policyGroupVersion) if err == nil { break } else if apierrors.IsTooManyRequests(err) { time.Sleep(5 * time.Second) } else { errCh <- fmt.Errorf("error when evicting pod %q: %v", pod.Name, err) return } } podArray := []api.Pod{pod} _, err = o.waitForDelete(podArray, kubectl.Interval, time.Duration(math.MaxInt64), true, getPodFn) if err == nil { doneCh <- true } else { errCh <- fmt.Errorf("error when waiting for pod %q terminating: %v", pod.Name, err) } }(pod, doneCh, errCh) } doneCount := 0 // 0 timeout means infinite, we use MaxInt64 to represent it. var globalTimeout time.Duration if o.Timeout == 0 { globalTimeout = time.Duration(math.MaxInt64) } else { globalTimeout = o.Timeout } for { select { case err := <-errCh: </ins> return err <ins> case <-doneCh: doneCount++ if doneCount == len(pods) { return nil } case <-time.After(globalTimeout): return fmt.Errorf("Drain did not complete within %v", globalTimeout) </ins> } } <ins> } </ins> <del> getPodFn := func(namespace, name string) (*api.Pod, error) { return o.client.Core().Pods(namespace).Get(name) </del> <ins> func (o *DrainOptions) deletePods(pods []api.Pod, getPodFn func(namespace, name string) (*api.Pod, error)) error { // 0 timeout means infinite, we use MaxInt64 to represent it. var globalTimeout time.Duration if o.Timeout == 0 { globalTimeout = time.Duration(math.MaxInt64) } else { globalTimeout = o.Timeout } for _, pod := range pods { err := o.deletePod(pod) if err != nil { return err } </ins> } <del> _, err = o.waitForDelete(pods, kubectl.Interval, o.Timeout, getPodFn) </del> <ins> _, err := o.waitForDelete(pods, kubectl.Interval, globalTimeout, false, getPodFn) </ins> return err } <del> func (o *DrainOptions) waitForDelete(pods []api.Pod, interval, timeout time.Duration, getPodFn func(string, string) (*api.Pod, error)) ([]api.Pod, error) { </del> <ins> func (o *DrainOptions) waitForDelete(pods []api.Pod, interval, timeout time.Duration, usingEviction bool, getPodFn func(string, string) (*api.Pod, error)) ([]api.Pod, error) { var verbStr string if usingEviction { verbStr = "evicted" } else { verbStr = "deleted" } </ins> err := wait.PollImmediate(interval, timeout, func() (bool, error) { pendingPods := []api.Pod{} for i, pod := range pods { p, err := getPodFn(pod.Namespace, pod.Name) if apierrors.IsNotFound(err) || (p != nil && p.ObjectMeta.UID != pod.ObjectMeta.UID) { <del> cmdutil.PrintSuccess(o.mapper, false, o.out, "pod", pod.Name, false, "deleted") </del> <ins> cmdutil.PrintSuccess(o.mapper, false, o.out, "pod", pod.Name, false, verbStr) </ins> continue } else if err != nil { return false, err

o.mapper, _ = f.Object() o.out = os.Stdout _, pods := createPods(false) <del> pendingPods, err := o.waitForDelete(pods, test.interval, test.timeout, test.getPodFn) </del> <ins> pendingPods, err := o.waitForDelete(pods, test.interval, test.timeout, false, test.getPodFn) </ins> if test.expectError { if err == nil {