Change:
<!-- Please only use this template for submitting reports about flaky tests or jobs (pass or fail with no underlying change in code) in Kubernetes CI --integration test jobs TestQuotaLimitService which was recently in One example run: I suspect this is because TestQuotaLimitService runs through multiple test cases where each test case relies on state created from the previous case. However, resource quota changes take some time to get propagated on the system and the test is trying to validate acceptance of new Services before the resource quota controller applies the necessary changes. links to appreciated links to specific failures in spyglass appreciated <!-- Please see the deflaking doc () for more guidance! --
cc
/sig network /sig apimachinery
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /sig network /sig apimachinery Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig api-machinery
Opened a fix here
/triage accepted

Discussion:
import ( "context" <ins> "errors" </ins> "fmt" "net/http" "net/http/httptest"

"time" v1 "k8s.io/api/core/v1" <ins> apierrors "k8s.io/apimachinery/pkg/api/errors" </ins> "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields"

"k8s.io/kubernetes/test/integration/framework" ) <ins> const ( resourceQuotaTimeout = 10 * time.Second ) </ins> // 1.2 code gets: // 	quota_test.go:95: Took 4.218619579s to scale up without quota // 	quota_test.go:199: unexpected error: timed out waiting for the condition, ended with 342 pods (1 minute)

} } <ins> // waitForUsedResourceQuota polls a ResourceQuota status for an expected used value func waitForUsedResourceQuota(t *testing.T, c clientset.Interface, ns, quotaName string, used v1.ResourceList) { err := wait.Poll(1*time.Second, resourceQuotaTimeout, func() (bool, error) { resourceQuota, err := c.CoreV1().ResourceQuotas(ns).Get(context.TODO(), quotaName, metav1.GetOptions{}) if err != nil { return false, err } // used may not yet be calculated if resourceQuota.Status.Used == nil { return false, nil } // verify that the quota shows the expected used resource values for k, v := range used { actualValue, found := resourceQuota.Status.Used[k] if !found { t.Logf("resource %s was not found in ResourceQuota status", k) return false, nil } if !actualValue.Equal(v) { t.Logf("resource %s, expected %s, actual %s", k, v.String(), actualValue.String()) return false, nil } } return true, nil }) if err != nil { t.Errorf("error waiting or ResourceQuota status: %v", err) } } </ins> func scale(t *testing.T, namespace string, clientset *clientset.Clientset) { target := int32(100) rc := &v1.ReplicationController{

func TestQuotaLimitService(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceLBNodePortControl, true)() <del> type testCase struct { description string svc         *v1.Service success     bool } </del> // Set up an API server h := &framework.APIServerHolder{Initialized: make(chan struct{})} s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

}, }, } waitForQuota(t, quota, clientset) <del> tests := []testCase{ { description: "node port service should be created successfully", svc:         newService("np-svc", v1.ServiceTypeNodePort, true), success:     true, }, { description: "first LB type service that allocates node port should be created successfully", svc:         newService("lb-svc-withnp1", v1.ServiceTypeLoadBalancer, true), success:     true, }, { description: "second LB type service that allocates node port creation should fail as node port quota is exceeded", svc:         newService("lb-svc-withnp2", v1.ServiceTypeLoadBalancer, true), success:     false, }, { description: "first LB type service that doesn't allocates node port should be created successfully", svc:         newService("lb-svc-wonp1", v1.ServiceTypeLoadBalancer, false), success:     true, }, { description: "second LB type service that doesn't allocates node port creation should fail as loadbalancer quota is exceeded", svc:         newService("lb-svc-wonp2", v1.ServiceTypeLoadBalancer, false), success:     false, }, { description: "forth service creation should be successful", svc:         newService("clusterip-svc1", v1.ServiceTypeClusterIP, false), success:     true, }, { description: "fifth service creation should fail as service quota is exceeded", svc:         newService("clusterip-svc2", v1.ServiceTypeClusterIP, false), success:     false, }, </del> <ins> // Creating the first node port service should succeed nodePortService := newService("np-svc", v1.ServiceTypeNodePort, true) _, err = clientset.CoreV1().Services(ns.Name).Create(context.TODO(), nodePortService, metav1.CreateOptions{}) if err != nil { t.Errorf("creating first node port Service should not have returned error: %v", err) </ins> } <del> for _, test := range tests { t.Log(test.description) _, err := clientset.CoreV1().Services(ns.Name).Create(context.TODO(), test.svc, metav1.CreateOptions{}) if (err == nil) != test.success { if err != nil { t.Fatalf("Error creating test service: %v, svc: %+v", err, test.svc) } else { t.Fatalf("Expect service creation to fail, but service %s is created", test.svc.Name) } </del> <ins> // Creating the first loadbalancer service should succeed lbServiceWithNodePort1 := newService("lb-svc-withnp1", v1.ServiceTypeLoadBalancer, true) _, err = clientset.CoreV1().Services(ns.Name).Create(context.TODO(), lbServiceWithNodePort1, metav1.CreateOptions{}) if err != nil { t.Errorf("creating first loadbalancer Service should not have returned error: %v", err) } // wait for ResourceQuota status to be updated before proceeding, otherwise the test will race with resource quota controller expectedQuotaUsed := v1.ResourceList{ v1.ResourceServices:              resource.MustParse("2"), v1.ResourceServicesNodePorts:     resource.MustParse("2"), v1.ResourceServicesLoadBalancers: resource.MustParse("1"), } waitForUsedResourceQuota(t, clientset, quota.Namespace, quota.Name, expectedQuotaUsed) // Creating another loadbalancer Service using node ports should fail because node prot quota is exceeded lbServiceWithNodePort2 := newService("lb-svc-withnp2", v1.ServiceTypeLoadBalancer, true) testServiceForbidden(clientset, ns.Name, lbServiceWithNodePort2, t) // Creating a loadbalancer Service without node ports should succeed lbServiceWithoutNodePort1 := newService("lb-svc-wonp1", v1.ServiceTypeLoadBalancer, false) _, err = clientset.CoreV1().Services(ns.Name).Create(context.TODO(), lbServiceWithoutNodePort1, metav1.CreateOptions{}) if err != nil { t.Errorf("creating another loadbalancer Service without node ports should not have returned error: %v", err) } // wait for ResourceQuota status to be updated before proceeding, otherwise the test will race with resource quota controller expectedQuotaUsed = v1.ResourceList{ v1.ResourceServices:              resource.MustParse("3"), v1.ResourceServicesNodePorts:     resource.MustParse("2"), v1.ResourceServicesLoadBalancers: resource.MustParse("2"), } waitForUsedResourceQuota(t, clientset, quota.Namespace, quota.Name, expectedQuotaUsed) // Creating another loadbalancer Service without node ports should fail because loadbalancer quota is exceeded lbServiceWithoutNodePort2 := newService("lb-svc-wonp2", v1.ServiceTypeLoadBalancer, false) testServiceForbidden(clientset, ns.Name, lbServiceWithoutNodePort2, t) // Creating a ClusterIP Service should succeed clusterIPService1 := newService("clusterip-svc1", v1.ServiceTypeClusterIP, false) _, err = clientset.CoreV1().Services(ns.Name).Create(context.TODO(), clusterIPService1, metav1.CreateOptions{}) if err != nil { t.Errorf("creating a cluster IP Service should not have returned error: %v", err) } // wait for ResourceQuota status to be updated before proceeding, otherwise the test will race with resource quota controller expectedQuotaUsed = v1.ResourceList{ v1.ResourceServices:              resource.MustParse("4"), v1.ResourceServicesNodePorts:     resource.MustParse("2"), v1.ResourceServicesLoadBalancers: resource.MustParse("2"), } waitForUsedResourceQuota(t, clientset, quota.Namespace, quota.Name, expectedQuotaUsed) // Creating a ClusterIP Service should fail because Service quota has been exceeded. clusterIPService2 := newService("clusterip-svc2", v1.ServiceTypeClusterIP, false) testServiceForbidden(clientset, ns.Name, clusterIPService2, t) } // testServiceForbidden attempts to create a Service expecting 403 Forbidden due to resource quota limits being exceeded. func testServiceForbidden(clientset clientset.Interface, namespace string, service *v1.Service, t *testing.T) { pollErr := wait.PollImmediate(2*time.Second, 30*time.Second, func() (bool, error) { _, err := clientset.CoreV1().Services(namespace).Create(context.TODO(), service, metav1.CreateOptions{}) if apierrors.IsForbidden(err) { return true, nil </ins> } <ins> if err == nil { return false, errors.New("creating Service should have returned error but got nil") } return false, nil }) if pollErr != nil { t.Errorf("creating Service should return Forbidden due to resource quota limits but got: %v", pollErr) </ins> } }