Change:
kubedrain has "onPodDeletedOrEvicted" callback that can be used in custom controllers. It would be very helpful if we add more callbacks that can give more granular control over monitoring, like "onPodDeletionOrEvicitionStarted" and "onPodDeletionOrEvictionFailed". It would help monitor the draining of nodes on a more granular level. Sometimes while draining, if the pod is stuck during the eviction or deletion of node, there is not much information that custom controllers can get from kubedrain, with the above-mentioned callbacks custom controllers will know which pod is evicting or if it failed during the eviction process.
/sig node
What is the custom controllers in your scenario? Can you elaborate on what you want to achieve with more details. Is this is kubectl improvement, the best sig to involve is sig cli. /sig cli
/remove-sig node please re-apply the sig node if this will end up being a request to sig node
In our case we have in CAPI (). Sometimes while draining, if the pod or volume is stuck during the eviction, there is not much information that we get from kubedrain.Helper. It would be much easier for the controller to report these kinds of issues with more granular callbacks like "onPodDeletionOrEvicitionStarted" and "onPodDeletionOrEvictionFailed" on kubedrain.Helper.
/assign Assigning this to myself, I have discussed this issue in sig-cli community meeting and they have given green light for this improvement.
/triage accepted /priority backlog

Discussion:
"k8s.io/cli-runtime/pkg/genericiooptions" "k8s.io/cli-runtime/pkg/printers" "k8s.io/cli-runtime/pkg/resource" <ins> "k8s.io/klog/v2" </ins> cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/drain" "k8s.io/kubectl/pkg/scheme"

ChunkSize:          cmdutil.DefaultChunkSize, }, } <del> o.drainer.OnPodDeletedOrEvicted = o.onPodDeletedOrEvicted </del> <ins> o.drainer.OnPodDeletionOrEvictionFinished = o.onPodDeletionOrEvictionFinished o.drainer.OnPodDeletionOrEvictionStarted = o.onPodDeletionOrEvictionStarted </ins> return o } <del> // onPodDeletedOrEvicted is called by drain.Helper, when the pod has been deleted or evicted func (o *DrainCmdOptions) onPodDeletedOrEvicted(pod *corev1.Pod, usingEviction bool) { </del> <ins> // onPodDeletionOrEvictionFinished is called by drain.Helper, when eviction/deletetion of the pod is finished func (o *DrainCmdOptions) onPodDeletionOrEvictionFinished(pod *corev1.Pod, usingEviction bool, err error) { </ins> var verbStr string if usingEviction { <del> verbStr = "evicted" </del> <ins> if err != nil { verbStr = "eviction failed" } else { verbStr = "evicted" } } else { if err != nil { verbStr = "deletion failed" } else { verbStr = "deleted" } } printObj, err := o.ToPrinter(verbStr) if err != nil { fmt.Fprintf(o.ErrOut, "error building printer: %vn", err) fmt.Fprintf(o.Out, "pod %s/%s %sn", pod.Namespace, pod.Name, verbStr) } else { _ = printObj(pod, o.Out) } } // onPodDeletionOrEvictionStarted is called by drain.Helper, when eviction/deletion of the pod is started func (o *DrainCmdOptions) onPodDeletionOrEvictionStarted(pod *corev1.Pod, usingEviction bool) { if !klog.V(2).Enabled() { return } var verbStr string if usingEviction { verbStr = "eviction started" </ins> } else { <del> verbStr = "deleted" </del> <ins> verbStr = "deletion started" </ins> } printObj, err := o.ToPrinter(verbStr) if err != nil { fmt.Fprintf(o.ErrOut, "error building printer: %vn", err) fmt.Fprintf(o.Out, "pod %s/%s %sn", pod.Namespace, pod.Name, verbStr) } else { <del> printObj(pod, o.Out) </del> <ins> _ = printObj(pod, o.Out) </ins> } }

DryRunStrategy cmdutil.DryRunStrategy // OnPodDeletedOrEvicted is called when a pod is evicted/deleted; for printing progress output <ins> // Deprecated: use OnPodDeletionOrEvictionFinished instead </ins> OnPodDeletedOrEvicted func(pod *corev1.Pod, usingEviction bool) <ins> // OnPodDeletionOrEvictionFinished is called when a pod is eviction/deletetion is failed; for printing progress output OnPodDeletionOrEvictionFinished func(pod *corev1.Pod, usingEviction bool, err error) // OnPodDeletionOrEvictionStarted is called when a pod eviction/deletion is started; for printing progress output OnPodDeletionOrEvictionStarted func(pod *corev1.Pod, usingEviction bool) </ins> } type waitForDeleteParams struct {

usingEviction                   bool getPodFn                        func(string, string) (*corev1.Pod, error) onDoneFn                        func(pod *corev1.Pod, usingEviction bool) <ins> onFinishFn                      func(pod *corev1.Pod, usingEviction bool, err error) </ins> globalTimeout                   time.Duration skipWaitForDeleteTimeoutSeconds int out                             io.Writer

case cmdutil.DryRunServer: fmt.Fprintf(d.Out, "evicting pod %s/%s (server dry run)n", pod.Namespace, pod.Name) default: <ins> if d.OnPodDeletionOrEvictionStarted != nil { d.OnPodDeletionOrEvictionStarted(&pod, true) } </ins> fmt.Fprintf(d.Out, "evicting pod %s/%sn", pod.Namespace, pod.Name) } select {

usingEviction:                   true, getPodFn:                        getPodFn, onDoneFn:                        d.OnPodDeletedOrEvicted, <ins> onFinishFn:                      d.OnPodDeletionOrEvictionFinished, </ins> globalTimeout:                   globalTimeout, skipWaitForDeleteTimeoutSeconds: d.SkipWaitForDeleteTimeoutSeconds, out:                             d.Out,

if err != nil && !apierrors.IsNotFound(err) { return err } <ins> if d.OnPodDeletionOrEvictionStarted != nil { d.OnPodDeletionOrEvictionStarted(&pod, false) } </ins> } ctx := d.getContext() params := waitForDeleteParams{

usingEviction:                   false, getPodFn:                        getPodFn, onDoneFn:                        d.OnPodDeletedOrEvicted, <ins> onFinishFn:                      d.OnPodDeletionOrEvictionFinished, </ins> globalTimeout:                   globalTimeout, skipWaitForDeleteTimeoutSeconds: d.SkipWaitForDeleteTimeoutSeconds, out:                             d.Out,

for i, pod := range pods { p, err := params.getPodFn(pod.Namespace, pod.Name) if apierrors.IsNotFound(err) || (p != nil && p.ObjectMeta.UID != pod.ObjectMeta.UID) { <del> if params.onDoneFn != nil { </del> <ins> if params.onFinishFn != nil { params.onFinishFn(&pod, params.usingEviction, nil) } else if params.onDoneFn != nil { </ins> params.onDoneFn(&pod, params.usingEviction) } continue } else if err != nil { <ins> if params.onFinishFn != nil { params.onFinishFn(&pod, params.usingEviction, err) } </ins> return false, err } else { if shouldSkipPod(*p, params.skipWaitForDeleteTimeoutSeconds) {

h := &Helper{ Out:                os.Stdout, GracePeriodSeconds: 10, <ins> OnPodDeletionOrEvictionStarted: func(pod *corev1.Pod, usingEviction bool) { if tc.evictionSupported && !tc.disableEviction { if !usingEviction { t.Errorf("%s: OnPodDeletionOrEvictionStarted callback failed while evicting; actualnt%vnexpectednt%v", tc.description, usingEviction, !usingEviction) } } else if tc.evictionSupported && tc.disableEviction { if usingEviction { t.Errorf("%s: OnPodDeletionOrEvictionStarted callback failed while deleting; actualnt%vnexpectednt%v", tc.description, !usingEviction, usingEviction) } } }, OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) { if tc.evictionSupported && !tc.disableEviction { if !usingEviction { t.Errorf("%s: OnPodDeletedOrEvicted callback failed while evicting; actualnt%vnexpectednt%v", tc.description, usingEviction, !usingEviction) } } else if tc.evictionSupported && tc.disableEviction { if usingEviction { t.Errorf("%s: OnPodDeletedOrEvicted callback failed while deleting; actualnt%vnexpectednt%v", tc.description, !usingEviction, usingEviction) } } }, OnPodDeletionOrEvictionFinished: func(pod *corev1.Pod, usingEviction bool, err error) { if tc.evictionSupported && !tc.disableEviction { if !usingEviction { t.Errorf("%s: OnPodDeletionOrEvictionFinished callback failed while evicting; actualnt%vnexpectednt%v", tc.description, usingEviction, !usingEviction) } } else if tc.evictionSupported && tc.disableEviction { if usingEviction { t.Errorf("%s: OnPodDeletionOrEvictionFinished callback failed while deleting; actualnt%vnexpectednt%v", tc.description, !usingEviction, usingEviction) } } }, </ins> } // Create 4 pods, and try to remove the first 2