Change:
The lock acquired by tryAcquireOrRenew is released when the leader ends leadership. However, due to the cancellation of the context, the lock may be set as an empty lock, so the Update cannot be run normally, resulting in a failure to release the lock. The lock can be released every time the leader ends. run this: and then you can got a log like these: and you when you watch the leases at the same time: you can see that was failed to release because of No response <details</details<details</details<details</details<details</details<details</detailsrelated packages: v1.3.0 v0.25.0 v0.25.0 v2.70.1
/sig api-machinery
/triage accepted thanks!
/unassign
/assign
/unassign
/unassign me

Discussion:
onRelease     = make(chan struct{}) lockObj runtime.Object <ins> gets    int </ins> updates int <ins> wg      sync.WaitGroup </ins> ) <ins> resetVars := func() { onNewLeader = make(chan struct{}) onRenewCalled = make(chan struct{}) onRenewResume = make(chan struct{}) onRelease = make(chan struct{}) </ins> <del> resourceLockConfig := rl.ResourceLockConfig{ Identity:      "baz", EventRecorder: &record.FakeRecorder{}, } c := &fake.Clientset{} c.AddReactor("get", objectType, func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { if lockObj != nil { return true, lockObj, nil } return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName()) }) // create lock c.AddReactor("create", objectType, func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { lockObj = action.(fakeclient.CreateAction).GetObject() return true, lockObj, nil }) c.AddReactor("update", objectType, func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { updates++ // Second update (first renew) should return our canceled error // FakeClient doesn't do anything with the context so we're doing this ourselves if updates == 2 { close(onRenewCalled) <-onRenewResume return true, nil, context.Canceled } else if updates == 3 { close(onRelease) } lockObj = action.(fakeclient.UpdateAction).GetObject() return true, lockObj, nil }) c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) { t.Errorf("unreachable action. testclient called too many times: %+v", action) return true, nil, fmt.Errorf("unreachable action") }) lock, err := rl.New(objectType, "foo", "bar", c.CoreV1(), c.CoordinationV1(), resourceLockConfig) if err != nil { t.Fatal("resourcelock.New() = ", err) </del> <ins> lockObj = nil gets = 0 updates = 0 </ins> } lec := LeaderElectionConfig{ <del> Lock:          lock, </del> LeaseDuration: 15 * time.Second, RenewDeadline: 2 * time.Second, RetryPeriod:   1 * time.Second,

}, } <del> elector, err := NewLeaderElector(lec) if err != nil { t.Fatal("Failed to create leader elector: ", err) </del> <ins> tests := []struct { name     string reactors []Reactor }{ { name: "release acquired lock on cancellation of update", reactors: []Reactor{ { verb:       "get", objectType: objectType, reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { gets++ if lockObj != nil { return true, lockObj, nil } return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName()) }, }, { verb:       "create", objectType: objectType, reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { lockObj = action.(fakeclient.CreateAction).GetObject() return true, lockObj, nil }, }, { verb:       "update", objectType: objectType, reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { updates++ // Second update (first renew) should return our canceled error // FakeClient doesn't do anything with the context so we're doing this ourselves if updates == 2 { close(onRenewCalled) <-onRenewResume return true, nil, context.Canceled } else if updates == 3 { // We update the lock after the cancellation to release it // This wg is to avoid the data race on lockObj defer wg.Done() close(onRelease) } lockObj = action.(fakeclient.UpdateAction).GetObject() return true, lockObj, nil }, }, }, }, { name: "release acquired lock on cancellation of get", reactors: []Reactor{ { verb:       "get", objectType: objectType, reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { gets++ if lockObj != nil { // Third and more get (first create, second renew) should return our canceled error // FakeClient doesn't do anything with the context so we're doing this ourselves if gets >= 3 { close(onRenewCalled) <-onRenewResume return true, nil, context.Canceled } return true, lockObj, nil } return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName()) }, }, { verb:       "create", objectType: objectType, reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { lockObj = action.(fakeclient.CreateAction).GetObject() return true, lockObj, nil }, }, { verb:       "update", objectType: objectType, reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { updates++ // Second update (first renew) should release the lock if updates == 2 { // We update the lock after the cancellation to release it // This wg is to avoid the data race on lockObj defer wg.Done() close(onRelease) } lockObj = action.(fakeclient.UpdateAction).GetObject() return true, lockObj, nil }, }, }, }, </ins> } <del> ctx, cancel := context.WithCancel(context.Background()) </del> <ins> for i := range tests { test := &tests[i] t.Run(test.name, func(t *testing.T) { wg.Add(1) resetVars() </ins> <del> go elector.Run(ctx) </del> <ins> resourceLockConfig := rl.ResourceLockConfig{ Identity:      "baz", EventRecorder: &record.FakeRecorder{}, } c := &fake.Clientset{} for _, reactor := range test.reactors { c.AddReactor(reactor.verb, objectType, reactor.reaction) } c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) { t.Errorf("unreachable action. testclient called too many times: %+v", action) return true, nil, fmt.Errorf("unreachable action") }) lock, err := rl.New(objectType, "foo", "bar", c.CoreV1(), c.CoordinationV1(), resourceLockConfig) if err != nil { t.Fatal("resourcelock.New() = ", err) } </ins> <del> // Wait for us to become the leader select { case <-onNewLeader: case <-time.After(10 * time.Second): t.Fatal("failed to become the leader") } </del> <ins> lec.Lock = lock elector, err := NewLeaderElector(lec) if err != nil { t.Fatal("Failed to create leader elector: ", err) } </ins> <del> // Wait for renew (update) to be invoked select { case <-onRenewCalled: case <-time.After(10 * time.Second): t.Fatal("the elector failed to renew the lock") } </del> <ins> ctx, cancel := context.WithCancel(context.Background()) </ins> <del> // Cancel the context - stopping the elector while // it's running cancel() </del> <ins> go elector.Run(ctx) </ins> <del> // Resume the update call to return the cancellation // which should trigger the release flow close(onRenewResume) </del> <ins> // Wait for us to become the leader select { case <-onNewLeader: case <-time.After(10 * time.Second): t.Fatal("failed to become the leader") } // Wait for renew (update) to be invoked select { case <-onRenewCalled: case <-time.After(10 * time.Second): t.Fatal("the elector failed to renew the lock") } </ins> <del> select { case <-onRelease: case <-time.After(10 * time.Second): t.Fatal("the lock was not released") </del> <ins> // Cancel the context - stopping the elector while // it's running cancel() // Resume the tryAcquireOrRenew call to return the cancellation // which should trigger the release flow close(onRenewResume) select { case <-onRelease: case <-time.After(10 * time.Second): t.Fatal("the lock was not released") } wg.Wait() }) </ins> } }

// Get returns the election record from a ConfigMap Annotation func (cml *configMapLock) Get(ctx context.Context) (*LeaderElectionRecord, []byte, error) { var record LeaderElectionRecord <del> var err error cml.cm, err = cml.Client.ConfigMaps(cml.ConfigMapMeta.Namespace).Get(ctx, cml.ConfigMapMeta.Name, metav1.GetOptions{}) </del> <ins> cm, err := cml.Client.ConfigMaps(cml.ConfigMapMeta.Namespace).Get(ctx, cml.ConfigMapMeta.Name, metav1.GetOptions{}) </ins> if err != nil { return nil, nil, err } <ins> cml.cm = cm </ins> if cml.cm.Annotations == nil { cml.cm.Annotations = make(map[string]string) }

// Get returns the election record from a Endpoints Annotation func (el *endpointsLock) Get(ctx context.Context) (*LeaderElectionRecord, []byte, error) { var record LeaderElectionRecord <del> var err error el.e, err = el.Client.Endpoints(el.EndpointsMeta.Namespace).Get(ctx, el.EndpointsMeta.Name, metav1.GetOptions{}) </del> <ins> ep, err := el.Client.Endpoints(el.EndpointsMeta.Namespace).Get(ctx, el.EndpointsMeta.Name, metav1.GetOptions{}) </ins> if err != nil { return nil, nil, err } <ins> el.e = ep </ins> if el.e.Annotations == nil { el.e.Annotations = make(map[string]string) }

// Get returns the election record from a Lease spec func (ll *LeaseLock) Get(ctx context.Context) (*LeaderElectionRecord, []byte, error) { <del> var err error ll.lease, err = ll.Client.Leases(ll.LeaseMeta.Namespace).Get(ctx, ll.LeaseMeta.Name, metav1.GetOptions{}) </del> <ins> lease, err := ll.Client.Leases(ll.LeaseMeta.Namespace).Get(ctx, ll.LeaseMeta.Name, metav1.GetOptions{}) </ins> if err != nil { return nil, nil, err } <ins> ll.lease = lease </ins> record := LeaseSpecToLeaderElectionRecord(&ll.lease.Spec) recordByte, err := json.Marshal(*record) if err != nil {