Change:
<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: The DaemonSet controller will schedule a pod on a node when the old pod is still in Terminating state. This breaks the expectation that only one pod will be running at a time per node. What you expected to happen: The DaemonSet controller should not let two pods running at the same time on the same node. How to reproduce it (as minimally and precisely as possible): Create a DaemonSet. Set terminationGracePeriodSeconds to be relatively long. Kill a pod. Watch a new pod start on the same node when the old pod is still terminating. Anything else we need to know?: Seems this behavior was introduced in Environment: Kubernetes version (use ): 1.7.2 Cloud provider or hardware configuration**: GKE OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others:
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig apps
/cc
cc
cc: I think the expected behavior is correct, the terminating pod could be performing an important step such as 'removing a lock' or 'cleanup' etc., we should allow that to gracefully die before creating a new one. IMHO we should revoke to old behavior, but why did we need this new behavior in the first place? That is not clear to me from the PR and Issue.
We don't currently guarantee at least or at most one scheduling for DaemonSet. Both behaviors are under consideration via issue and in the future work section of the . The reason for is to "Make sure that controllers are not adopting children that are marked for deletion" (). This is necessary for GC to work properly.
Well the behavior may be technically undefined, but can I request that in the future changes like this be mentioned in the release notes?
Yes, certainly. Also "The DaemonSet controller should not let two pods running at the same time on the same node." is not necessarily the same thing as respecting termination prior to launching a new Pod. The former implies a more strict requirement than the latter. I think we should consider at least attempting to wait for a terminating Pod to be deleted prior to creating another Pod on the Node for same DaemonSet as default behavior.
will something like work then?

Discussion:
} // If any adoptions are attempted, we should first recheck for deletion with // an uncached quorum read sometime after listing Pods (see #42639). <del> canAdoptFunc := controller.RecheckDeletionTimestamp(func() (metav1.Object, error) { </del> <ins> dsNotDeleted := controller.RecheckDeletionTimestamp(func() (metav1.Object, error) { </ins> fresh, err := dsc.kubeClient.ExtensionsV1beta1().DaemonSets(ds.Namespace).Get(ds.Name, metav1.GetOptions{}) if err != nil { return nil, err

} return fresh, nil }) // Use ControllerRefManager to adopt/orphan as needed. <del> cm := controller.NewPodControllerRefManager(dsc.podControl, ds, selector, controllerKind, canAdoptFunc) </del> <ins> cm := controller.NewPodControllerRefManager(dsc.podControl, ds, selector, controllerKind, dsNotDeleted) </ins> return cm.ClaimPods(pods) }

// Group Pods by Node name. nodeToDaemonPods := make(map[string][]*v1.Pod) for _, pod := range claimedPods { <del> // Skip terminating pods if pod.DeletionTimestamp != nil { continue } </del> nodeName := pod.Spec.NodeName nodeToDaemonPods[nodeName] = append(nodeToDaemonPods[nodeName], pod) }

// If there's non-daemon pods left on this node, we will create it in the next sync loop var daemonPodsRunning []*v1.Pod for _, pod := range daemonPods { <ins> if pod.DeletionTimestamp != nil { continue } </ins> if pod.Status.Phase == v1.PodFailed { msg := fmt.Sprintf("Found failed daemon pod %s/%s on node %s, will try to kill it", pod.Namespace, node.Name, pod.Name) glog.V(2).Infof(msg)

ds.Spec.UpdateStrategy = *strategy setDaemonSetToleration(ds, noScheduleTolerations) manager, podControl, _ := newTestController(ds) <del> node := newNode("untainted", nil) manager.nodeStore.Add(node) </del> <ins> addNodes(manager.nodeStore, 0, 1, nil) </ins> manager.dsStore.Add(ds) syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } } <ins> // DaemonSet should launch a pod on a not ready node with taint notReady:NoExecute. func TestDaemonSetRespectsTermination(t *testing.T) { for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy manager, podControl, _ := newTestController(ds) addNodes(manager.nodeStore, 0, 1, simpleNodeLabel) pod := newPod(fmt.Sprintf("%s-", "node-0"), "node-0", simpleDaemonSetLabel, ds) dt := metav1.Now() pod.DeletionTimestamp = &dt manager.podStore.Add(pod) manager.dsStore.Add(ds) syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } } </ins> func setNodeTaint(node *v1.Node, taints []v1.Taint) { node.Spec.Taints = taints }

newPod("non-matching-owned-0-", "node-0", simpleDaemonSetLabel2, ds), newPod("non-matching-orphan-1-", "node-1", simpleDaemonSetLabel2, nil), newPod("matching-owned-by-other-0-", "node-0", simpleDaemonSetLabel, ds2), <del> func() *v1.Pod { pod := newPod("matching-owned-2-but-set-for-deletion", "node-2", simpleDaemonSetLabel, ds) now := metav1.Now() pod.DeletionTimestamp = &now return pod }(), </del> } for _, pod := range ignoredPods { manager.podStore.Add(pod)

} available := false for _, pod := range daemonPods { <del> if podutil.IsPodAvailable(pod, ds.Spec.MinReadySeconds, metav1.Now()) { </del> <ins> //for the purposes of update we ensure that the Pod is both available and not terminating if podutil.IsPodAvailable(pod, ds.Spec.MinReadySeconds, metav1.Now()) && pod.DeletionTimestamp == nil { </ins> available = true break }

"k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" <ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" </ins> "k8s.io/apimachinery/pkg/util/intstr" )

maxUnavailable: 1, numUnavailable: 0, }, <ins> { name: "Two nodes with pods, MaxUnavailable in percents, pod terminating", Manager: func() *daemonSetsController { manager, _, _ := newTestController() addNodes(manager.nodeStore, 0, 2, nil) return manager }(), ds: func() *extensions.DaemonSet { ds := newDaemonSet("x") intStr := intstr.FromString("50%") ds.Spec.UpdateStrategy.RollingUpdate = &extensions.RollingUpdateDaemonSet{MaxUnavailable: &intStr} return ds }(), nodeToPods: func() map[string][]*v1.Pod { mapping := make(map[string][]*v1.Pod) pod0 := newPod("pod-0", "node-0", simpleDaemonSetLabel, nil) pod1 := newPod("pod-1", "node-1", simpleDaemonSetLabel, nil) now := metav1.Now() markPodReady(pod0) markPodReady(pod1) pod1.DeletionTimestamp = &now mapping["node-0"] = []*v1.Pod{pod0} mapping["node-1"] = []*v1.Pod{pod1} return mapping }(), maxUnavailable: 1, numUnavailable: 1, }, </ins> } for _, c := range cases {