Change:
In federation tests, if we observe the build logs. we see cleaning up LoadBalancer from cloud provider seems not working. It has 2 effects: delays the tests by waiting to delete but always returns error. so we waste like 5 minutes. maybe we are leaking the LoadBalancer in the project.
we can see these errors in every build. but just for reference here are some build logs https://k8s-
cc
All the errors in logs are . It doesn't necessarily indicate we are leaking resource, just now we must have clusterID initialized before calling EnsureLoadBalancerDeleted(). Would be better to not call this function directly in e2e test :)
Federation cleanup should use the same implementation as non-federated e2e test:
function looks like only specific to GCE. in future we would run the e2e tests in AWS too. it would be better to use generic functions if any.
using is working for us. here is the test logs https://k8s-
EnsureLoadBalancerResourcesDeleted() actually cleans up nothing, it only lists forwarding rules and probably logging :)

Discussion:
return fmt.Errorf("waiting for service to be down timed out") } <del> func CleanupServiceGCEResources(loadBalancerName string) { </del> <ins> func CleanupServiceResources(loadBalancerName, zone string) { if TestContext.Provider == "gce" || TestContext.Provider == "gke" { CleanupServiceGCEResources(loadBalancerName, zone) } // TODO: we need to add this function with other cloud providers, if there is a need. } func CleanupServiceGCEResources(loadBalancerName, zone string) { </ins> if pollErr := wait.Poll(5*time.Second, LoadBalancerCleanupTimeout, func() (bool, error) { <del> if err := CleanupGCEResources(loadBalancerName); err != nil { </del> <ins> if err := CleanupGCEResources(loadBalancerName, zone); err != nil { </ins> Logf("Still waiting for glbc to cleanup: %v", err) return false, nil }

// CleanupGCEResources cleans up GCE Service Type=LoadBalancer resources with // the given name. The name is usually the UUID of the Service prefixed with an // alpha-numeric character ('a') to work around cloudprovider rules. <del> func CleanupGCEResources(loadBalancerName string) (retErr error) { </del> <ins> func CleanupGCEResources(loadBalancerName, zone string) (retErr error) { </ins> gceCloud, err := GetGCECloud() if err != nil { return err } <ins> region, err := gcecloud.GetGCERegion(zone) if err != nil { return fmt.Errorf("error parsing GCE/GKE region from zone %q: %v", zone, err) } </ins> if err := gceCloud.DeleteFirewall(loadBalancerName); err != nil && !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { retErr = err } <del> if err := gceCloud.DeleteRegionForwardingRule(loadBalancerName, gceCloud.Region()); err != nil && </del> <ins> if err := gceCloud.DeleteRegionForwardingRule(loadBalancerName, region); err != nil && </ins> !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { retErr = fmt.Errorf("%vn%v", retErr, err) } <del> if err := gceCloud.DeleteRegionAddress(loadBalancerName, gceCloud.Region()); err != nil && </del> <ins> if err := gceCloud.DeleteRegionAddress(loadBalancerName, region); err != nil && </ins> !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { retErr = fmt.Errorf("%vn%v", retErr, err) }

if hc != nil { hcNames = append(hcNames, hc.Name) } <del> if err := gceCloud.DeleteExternalTargetPoolAndChecks(loadBalancerName, gceCloud.Region(), hcNames...); err != nil && </del> <ins> if err := gceCloud.DeleteExternalTargetPoolAndChecks(loadBalancerName, region, hcNames...); err != nil && </ins> !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { retErr = fmt.Errorf("%vn%v", retErr, err) }

} for _, lb := range serviceLBNames { framework.Logf("cleaning gce resource for %s", lb) <del> framework.CleanupServiceGCEResources(lb) </del> <ins> framework.CleanupServiceGCEResources(lb, framework.TestContext.CloudConfig.Zone) </ins> } //reset serviceLBNames serviceLBNames = []string{}

"//pkg/api/v1:go_default_library", "//pkg/apis/extensions/v1beta1:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", <ins> "//pkg/cloudprovider:go_default_library", </ins> "//test/e2e/chaosmonkey:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library",

import ( "fmt" <ins> "strings" </ins> "time" "k8s.io/apimachinery/pkg/api/errors"

framework.ExpectNoError(err, fmt.Sprintf("Error creating client for cluster %q: %+v", clusterConf.name, err)) return restConfig } <ins> func GetZoneFromClusterName(clusterName string) string { // Ref: https://github.com/kubernetes/kubernetes/blob/master/cluster/kube-util.sh#L55 prefix := "federation-e2e-" + framework.TestContext.Provider + "-" return strings.TrimPrefix(clusterName, prefix) } </ins>

"k8s.io/apimachinery/pkg/util/wait" federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1" fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" <del> "k8s.io/kubernetes/pkg/api" </del> "k8s.io/kubernetes/pkg/api/v1" kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" <ins> "k8s.io/kubernetes/pkg/cloudprovider" </ins> "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" fedframework "k8s.io/kubernetes/test/e2e_federation/framework"

continue } <ins> if cSvc.Spec.Type == v1.ServiceTypeLoadBalancer { // In federation tests, e2e zone names are used to derive federation member cluster names zone := fedframework.GetZoneFromClusterName(name) serviceLBName := cloudprovider.GetLoadBalancerName(cSvc) framework.Logf("cleaning cloud provider resource for service %q in namespace %q, in cluster %q", service.Name, namespace, name) framework.CleanupServiceResources(serviceLBName, zone) } </ins> err = cleanupServiceShard(c.Clientset, name, namespace, cSvc, fedframework.FederatedDefaultTestTimeout) if err != nil { framework.Logf("Failed to delete service %q in namespace %q, in cluster %q: %v", service.Name, namespace, name, err) } <del> if service.Spec.Type == v1.ServiceTypeLoadBalancer { err = cleanupServiceShardLoadBalancer(name, cSvc, fedframework.FederatedDefaultTestTimeout) if err != nil { framework.Logf("Failed to delete cloud provider resources for service %q in namespace %q, in cluster %q, err: %v", service.Name, namespace, name, err) } } </del> } }

return err } <del> func cleanupServiceShardLoadBalancer(clusterName string, service *v1.Service, timeout time.Duration) error { provider := framework.TestContext.CloudConfig.Provider if provider == nil { return fmt.Errorf("cloud provider undefined") } internalSvc := &v1.Service{} err := api.Scheme.Convert(service, internalSvc, nil) if err != nil { return fmt.Errorf("failed to convert versioned service object to internal type: %v", err) } err = wait.PollImmediate(framework.Poll, timeout, func() (bool, error) { lbi, supported := provider.LoadBalancer() if !supported { return false, fmt.Errorf("%q doesn't support load balancers", provider.ProviderName()) } err := lbi.EnsureLoadBalancerDeleted(clusterName, internalSvc) if err != nil { // Deletion failed with an error, try again. framework.Logf("Failed to delete cloud provider resources for service %q in namespace %q, in cluster %q: %v", service.Name, service.Namespace, clusterName, err) return false, nil } By(fmt.Sprintf("Cloud provider resources for Service %q in namespace %q in cluster %q deleted", service.Name, service.Namespace, clusterName)) return true, nil }) return err } </del> func podExitCodeDetector(f *fedframework.Framework, name, namespace string, code int32) func() error { // If we ever get any container logs, stash them here. logs := ""