Change:
Some external controller deleted node object while kubelet was alive Due to ignoring errors in kubelet's lease logic, it created lease without ownerReference set Kubelet doesn't create lease without ownerReference set in any case, even pathological one Some external controller deletes node object Garbage collector properly deletes associated lease object Kubelet tries to update object and it fails on Conflict (saying that UID doesn't match): We go to branch, call backoffEnsureLease and assign result to base, but backoffEnsureLease may return potentially invalid object if ensureLease returned error (which happens when newLeasePostProcessFunc returns error, e.g. when node doesn't exist). We The next loop calls c.newLease on the incorrect base and ignores error in line 181 (so the second attempt to call newLeasePostProcessFunc which also fails with an error) We do UPDATE with incomplete (without effect of newLeasePostProcessFunc) object Given that leases allow CreateOnUpdate a new object, without ownerReference is created: No response <details21.10 </details<details</details<details</details<details</details<details</details<details</details
/sig node
/assign
/cc
/triage accepted need to understand what is the right behavior here
We are in trouble for this problem too. I think It’s exactly as said. Lease has the special update endpoint which can create the new lease objects not only update them. If it receives a request with a new lease object which has non-nil UID field, returns a conflict error. This situation can be occured in the function when a lease object is deleted. The lease object is deleted when the node resource is deleted in the result of GC. When received a conflict error in updating a lease object and the parent node object doesn't exist, , and continue next loop. In the next loop, the function returns the new lease object without ownerReference and UID. As a result, it is created unexpectively by the special update endpoint.
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle stale
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
/remove-lifecycle rotten

Discussion:
latestLease *coordinationv1.Lease // newLeasePostProcessFunc allows customizing a lease object (e.g. setting OwnerReference) <del> // before every time the lease is created/refreshed(updated). Note that an error will block // a lease CREATE, causing the controller to retry next time, but an error won't block a // lease UPDATE. </del> <ins> // before every time the lease is created/refreshed(updated). // Note that an error will block the lease operation. </ins> newLeasePostProcessFunc ProcessLeaseFunc }

// call this once you're sure the lease has been created func (c *controller) retryUpdateLease(ctx context.Context, base *coordinationv1.Lease) error { for i := 0; i < maxUpdateRetries; i++ { <del> leaseToUpdate, _ := c.newLease(base) lease, err := c.leaseClient.Update(ctx, leaseToUpdate, metav1.UpdateOptions{}) if err == nil { c.latestLease = lease return nil } klog.FromContext(ctx).Error(err, "Failed to update lease") // OptimisticLockError requires getting the newer version of lease to proceed. if apierrors.IsConflict(err) { base, _ = c.backoffEnsureLease(ctx) continue </del> <ins> leaseToUpdate, err := c.newLease(base) if err != nil { klog.FromContext(ctx).Error(err, "Failed to prepare lease") } else { lease, err := c.leaseClient.Update(ctx, leaseToUpdate, metav1.UpdateOptions{}) if err == nil { c.latestLease = lease return nil } klog.FromContext(ctx).Error(err, "Failed to update lease") // OptimisticLockError requires getting the newer version of lease to proceed. if apierrors.IsConflict(err) { base, _ = c.backoffEnsureLease(ctx) continue } </ins> } if i > 0 && c.onRepeatedHeartbeatFailure != nil { c.onRepeatedHeartbeatFailure()

getReactor                 func(action clienttesting.Action) (bool, runtime.Object, error) onRepeatedHeartbeatFailure func() expectErr                  bool <ins> client                     *fake.Clientset </ins> }{ { desc: "no errors",

getReactor:                 nil, onRepeatedHeartbeatFailure: nil, expectErr:                  false, <ins> client:                     fake.NewSimpleClientset(node), </ins> }, { desc: "connection errors",

getReactor:                 nil, onRepeatedHeartbeatFailure: nil, expectErr:                  true, <ins> client:                     fake.NewSimpleClientset(node), </ins> }, { desc: "optimistic lock errors",

}, onRepeatedHeartbeatFailure: func() { t.Fatalf("onRepeatedHeartbeatFailure called") }, expectErr:                  false, <ins> client:                     fake.NewSimpleClientset(node), }, { desc: "node not found errors", updateReactor: func(action clienttesting.Action) (bool, runtime.Object, error) { t.Fatalf("lease was updated when node does not exist!") return true, nil, nil }, getReactor:                 nil, onRepeatedHeartbeatFailure: nil, expectErr:                  true, client:                     fake.NewSimpleClientset(), </ins> }, } for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { _, ctx := ktesting.NewTestContext(t) <del> cl := fake.NewSimpleClientset(node) </del> <ins> cl := tc.client </ins> if tc.updateReactor != nil { cl.PrependReactor("update", "leases", tc.updateReactor) }