Change:
/kind bug /sig network What happened: Networking tight loop causing GCP Cloud rate limiting to kick in. Causing tests to fail. Suspected call stack: Looks like it could be the root cause of these test failures here: https://k8s- These storage tests are hitting rate limit on the cloud api when trying to detach/delete disks. However, we have exponential backoff for all these functions. But the rate limiting from an external source is affecting the tests. See logs snippet: Note: These logs span exactly 1 minute. Try grepping "rateLimit" or "CIDRNotAvailable" -also note that the CIDRNotAvailable error messages are probably not verbose enough as they do not pinpoint what the actual error is, you can only associate them by correlation in timing and then looking at the code. Shorter snippet from above log file to demonstrate: since this is running on hundreds of nodes hitting this API call should involve some sort of exponential backoff as well as jitter. /cc /assign
/assign
/assign
/sig cloud-provider /sig gcp /milestone v1.12 (since is marked as v1.12)
may we have a priority on this since Code Freeze is next week? :)
Quick question, are we using the default API rate limits quota for scalability projects?
I will defer to for priority because he probably has the most context on how serious this is and what it could affect
Which rate limit are you referring to? (I don't remember us changing it any time) Btw - this is the project being used for that job -
I put up a patch for adding exponential retry
Gentle reminder that for this to be considered for 1.12 past Code Freeze, which is next week, it must have a status/critical-urgent label. I see this is being actively worked on

Discussion:
cidrUpdateRetries = 3 // updateRetryTimeout is the time to wait before requeing a failed node for retry <del> updateRetryTimeout = 100 * time.Millisecond </del> <ins> updateRetryTimeout = 250 * time.Millisecond // maxUpdateRetryTimeout is the maximum amount of time between timeouts. maxUpdateRetryTimeout = 5 * time.Second </ins> // updateMaxRetries is the max retries for a failed node updateMaxRetries = 10

"github.com/golang/glog" <ins> "k8s.io/api/core/v1" </ins> "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types"

"k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" <del> "k8s.io/api/core/v1" </del> clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1"

glog.Warning("Channel nodeCIDRUpdateChannel was unexpectedly closed") return } <del> if err := ca.updateCIDRAllocation(workItem); err != nil { if ca.canRetry(workItem) { time.AfterFunc(updateRetryTimeout, func() { </del> <ins> if err := ca.updateCIDRAllocation(workItem); err == nil { glog.V(3).Infof("Updated CIDR for %q", workItem) ca.removeNodeFromProcessing(workItem) } else { glog.Errorf("Error updating CIDR for %q: %v", workItem, err) if canRetry, timeout := ca.retryParams(workItem); canRetry { glog.V(2).Infof("Retrying update for %q after %v", workItem, timeout) time.AfterFunc(timeout, func() { </ins> // Requeue the failed node for update again. ca.nodeUpdateChannel <- workItem }) continue } <ins> glog.Errorf("Exceeded retry count for %q, dropping from queue", workItem) </ins> } <del> ca.removeNodeFromProcessing(workItem) </del> case <-stopChan: return }

return true } <del> func (ca *cloudCIDRAllocator) canRetry(nodeName string) bool { </del> <ins> func (ca *cloudCIDRAllocator) retryParams(nodeName string) (bool, time.Duration) { </ins> ca.lock.Lock() defer ca.lock.Unlock() <del> count := ca.nodesInProcessing[nodeName].retries + 1 </del> <ins> entry, ok := ca.nodesInProcessing[nodeName] if !ok { glog.Errorf("Cannot get retryParams for %q as entry does not exist", nodeName) return false, 0 } count := entry.retries + 1 </ins> if count > updateMaxRetries { <del> return false </del> <ins> return false, 0 </ins> } ca.nodesInProcessing[nodeName].retries = count <del> return true </del> <ins> return true, nodeUpdateRetryTimeout(count) } func nodeUpdateRetryTimeout(count int) time.Duration { timeout := updateRetryTimeout for i := 0; i < count && timeout < maxUpdateRetryTimeout; i++ { timeout *= 2 } if timeout > maxUpdateRetryTimeout { return maxUpdateRetryTimeout } return timeout </ins> } func (ca *cloudCIDRAllocator) removeNodeFromProcessing(nodeName string) {

package ipam import ( <ins> "fmt" </ins> "testing" "time"

// wait for node to finish processing (should terminate and not time out) } } <ins> func TestNodeUpdateRetryTimeout(t *testing.T) { for _, tc := range []struct { count int want  time.Duration }{ {count: 0, want: 250 * time.Millisecond}, {count: 1, want: 500 * time.Millisecond}, {count: 2, want: 1000 * time.Millisecond}, {count: 3, want: 2000 * time.Millisecond}, {count: 50, want: 5000 * time.Millisecond}, } { t.Run(fmt.Sprintf("count %d", tc.count), func(t *testing.T) { if got := nodeUpdateRetryTimeout(tc.count); got != tc.want { t.Errorf("nodeUpdateRetryTimeout(tc.count) = %v; want %v", got, tc.want) } }) } } </ins>