Repository: kubernetes-sigs/cluster-api-provider-aws

Issue:
Parallel creation of multiple loadbalancers
/kind feature

**Describe the solution you'd like**
LoadBalancers take many minutes to become available after creation. We currently wait synchronously for a loadbalancer to become available immediately after creation: https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/63b8bcf1b3860347d6f860a3dd5bd582d1899a95/pkg/cloud/services/elb/loadbalancer.go#L128-L133

This is inefficient when creating more than one ControlPlaneLoadBalancer, for example when creating both the internal and external LBs of an OpenShift cluster. Ideally we would create all loadbalancers first, then wait for them to become ready.

**Anything else you would like to add:**
[Miscellaneous information that will assist in solving the issue.]


**Environment:**

- Cluster-api-provider-aws version: v2.8.1-129-g63b8bcf1b (main)

Hints from issue comments:
@k8s-ci-robot: This issue is currently awaiting triage.

If CAPA/CAPI contributors determines this is a relevant issue, they will accept it by applying the `triage/accepted` label and provide further guidance.

The `triage/accepted` label can be added by org members by writing `/triage accepted` in a comment.


<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>

Solution patch:
diff --git a/pkg/cloud/services/elb/loadbalancer.go b/pkg/cloud/services/elb/loadbalancer.go
index 5af91b26c..c2265012a 100644
--- a/pkg/cloud/services/elb/loadbalancer.go
+++ b/pkg/cloud/services/elb/loadbalancer.go
@@ -67,11 +67,21 @@ const additionalTargetGroupPrefix = "additional-listener-"
 // cantAttachSGToNLBRegions is a set of regions that do not support Security Groups in NLBs.
 var cantAttachSGToNLBRegions = sets.New("us-iso-east-1", "us-iso-west-1", "us-isob-east-1")
 
+type lbReconciler func() error
+
 // ReconcileLoadbalancers reconciles the load balancers for the given cluster.
 func (s *Service) ReconcileLoadbalancers() error {
 	s.scope.Debug("Reconciling load balancers")
 
 	var errs []error
+	var lbReconcilers []lbReconciler
+
+	// The following splits load balancer reconciliation into 2 phases:
+	// 1. Get or create the load balancer
+	// 2. Reconcile the load balancer
+	// We ensure that we only wait for the load balancer to become available in
+	// the reconcile phase. This is useful when creating multiple load
+	// balancers, as they can take several minutes to become available.
 
 	for _, lbSpec := range s.scope.ControlPlaneLoadBalancers() {
 		if lbSpec == nil {
@@ -79,48 +89,72 @@ func (s *Service) ReconcileLoadbalancers() error {
 		}
 		switch lbSpec.LoadBalancerType {
 		case infrav1.LoadBalancerTypeClassic:
-			errs = append(errs, s.reconcileClassicLoadBalancer())
+			reconciler, err := s.getOrCreateClassicLoadBalancer()
+			if err != nil {
+				errs = append(errs, err)
+			} else {
+				lbReconcilers = append(lbReconcilers, reconciler)
+			}
 		case infrav1.LoadBalancerTypeNLB, infrav1.LoadBalancerTypeALB, infrav1.LoadBalancerTypeELB:
-			errs = append(errs, s.reconcileV2LB(lbSpec))
+			reconciler, err := s.getOrCreateV2LB(lbSpec)
+			if err != nil {
+				errs = append(errs, err)
+			} else {
+				lbReconcilers = append(lbReconcilers, reconciler)
+			}
 		default:
 			errs = append(errs, fmt.Errorf("unknown or unsupported load balancer type on primary load balancer: %s", lbSpec.LoadBalancerType))
 		}
 	}
 
+	// Reconcile all load balancers
+	for _, reconciler := range lbReconcilers {
+		if err := reconciler(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
 	return kerrors.NewAggregate(errs)
 }
 
-// reconcileV2LB creates a load balancer. It also takes care of generating unique names across
-// namespaces by appending the namespace to the name.
-func (s *Service) reconcileV2LB(lbSpec *infrav1.AWSLoadBalancerSpec) error {
+// getOrCreateV2LB gets an existing load balancer, or creates a new one if it does not exist.
+// It also takes care of generating unique names across namespaces by appending the namespace to the name.
+// It returns a function that reconciles the load balancer.
+func (s *Service) getOrCreateV2LB(lbSpec *infrav1.AWSLoadBalancerSpec) (lbReconciler, error) {
 	name, err := LBName(s.scope, lbSpec)
 	if err != nil {
-		return errors.Wrap(err, "failed to get control plane load balancer name")
+		return nil, errors.Wrap(err, "failed to get control plane load balancer name")
 	}
 
 	// Get default api server spec.
 	desiredLB, err := s.getAPIServerLBSpec(name, lbSpec)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	lb, err := s.describeLB(name, lbSpec)
 	switch {
 	case IsNotFound(err) && s.scope.ControlPlaneEndpoint().IsValid():
 		// if elb is not found and owner cluster ControlPlaneEndpoint is already populated, then we should not recreate the elb.
-		return errors.Wrapf(err, "no loadbalancer exists for the AWSCluster %s, the cluster has become unrecoverable and should be deleted manually", s.scope.InfraClusterName())
+		return nil, errors.Wrapf(err, "no loadbalancer exists for the AWSCluster %s, the cluster has become unrecoverable and should be deleted manually", s.scope.InfraClusterName())
 	case IsNotFound(err):
 		lb, err = s.createLB(desiredLB, lbSpec)
 		if err != nil {
 			s.scope.Error(err, "failed to create LB")
-			return err
+			return nil, err
 		}
 
 		s.scope.Debug("Created new network load balancer for apiserver", "api-server-lb-name", lb.Name)
 	case err != nil:
 		// Failed to describe the classic ELB
-		return err
+		return nil, err
 	}
 
+	return func() error {
+		return s.reconcileV2LB(lb, desiredLB, lbSpec)
+	}, nil
+}
+
+func (s *Service) reconcileV2LB(lb *infrav1.LoadBalancer, desiredLB *infrav1.LoadBalancer, lbSpec *infrav1.AWSLoadBalancerSpec) error {
 	wReq := &elbv2.DescribeLoadBalancersInput{
 		LoadBalancerArns: aws.StringSlice([]string{lb.ARN}),
 	}
@@ -442,12 +476,21 @@ func (s *Service) createLB(spec *infrav1.LoadBalancer, lbSpec *infrav1.AWSLoadBa
 
 	// Target Groups and listeners will be reconciled separately
 
-	s.scope.Info("Created network load balancer", "dns-name", *out.LoadBalancers[0].DNSName)
+	if out.LoadBalancers[0].DNSName == nil {
+		return nil, fmt.Errorf("CreateLoadBalancer did not return a DNS name for %s", spec.Name)
+	}
+	dnsName := *out.LoadBalancers[0].DNSName
+	if out.LoadBalancers[0].LoadBalancerArn == nil {
+		return nil, fmt.Errorf("CreateLoadBalancer did not return an ARN for %s", spec.Name)
+	}
+	arn := *out.LoadBalancers[0].LoadBalancerArn
+
+	s.scope.Info("Created network load balancer", "dns-name", dnsName)
 
 	res := spec.DeepCopy()
-	s.scope.Debug("applying load balancer DNS to result", "dns", *out.LoadBalancers[0].DNSName)
-	res.DNSName = *out.LoadBalancers[0].DNSName
-	res.ARN = *out.LoadBalancers[0].LoadBalancerArn
+	s.scope.Debug("applying load balancer DNS to result", "dns", dnsName)
+	res.DNSName = dnsName
+	res.ARN = arn
 	return res, nil
 }
 
@@ -507,37 +550,46 @@ func (s *Service) describeLB(name string, lbSpec *infrav1.AWSLoadBalancerSpec) (
 	return fromSDKTypeToLB(out.LoadBalancers[0], outAtt.Attributes, tags), nil
 }
 
-func (s *Service) reconcileClassicLoadBalancer() error {
+// getOrCreateClassicLoadBalancer gets an existing classic load balancer, or creates a new one if it does not exist.
+// It also takes care of generating unique names across namespaces by appending the namespace to the name.
+// It returns a function that reconciles the load balancer.
+func (s *Service) getOrCreateClassicLoadBalancer() (lbReconciler, error) {
 	// Generate a default control plane load balancer name. The load balancer name cannot be
 	// generated by the defaulting webhook, because it is derived from the cluster name, and that
 	// name is undefined at defaulting time when generateName is used.
 	name, err := ELBName(s.scope)
 	if err != nil {
-		return errors.Wrap(err, "failed to get control plane load balancer name")
+		return nil, errors.Wrap(err, "failed to get control plane load balancer name")
 	}
 
 	// Get default api server spec.
 	spec, err := s.getAPIServerClassicELBSpec(name)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	apiELB, err := s.describeClassicELB(spec.Name)
 	switch {
 	case IsNotFound(err) && s.scope.ControlPlaneEndpoint().IsValid():
 		// if elb is not found and owner cluster ControlPlaneEndpoint is already populated, then we should not recreate the elb.
-		return errors.Wrapf(err, "no loadbalancer exists for the AWSCluster %s, the cluster has become unrecoverable and should be deleted manually", s.scope.InfraClusterName())
+		return nil, errors.Wrapf(err, "no loadbalancer exists for the AWSCluster %s, the cluster has become unrecoverable and should be deleted manually", s.scope.InfraClusterName())
 	case IsNotFound(err):
 		apiELB, err = s.createClassicELB(spec)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		s.scope.Debug("Created new classic load balancer for apiserver", "api-server-elb-name", apiELB.Name)
 	case err != nil:
 		// Failed to describe the classic ELB
-		return err
+		return nil, err
 	}
 
+	return func() error {
+		return s.reconcileClassicLoadBalancer(apiELB, spec)
+	}, nil
+}
+
+func (s *Service) reconcileClassicLoadBalancer(apiELB *infrav1.LoadBalancer, spec *infrav1.LoadBalancer) error {
 	if apiELB.IsManaged(s.scope.Name()) {
 		if !cmp.Equal(spec.ClassicElbAttributes, apiELB.ClassicElbAttributes) {
 			err := s.configureAttributes(apiELB.Name, spec.ClassicElbAttributes)
@@ -546,6 +598,9 @@ func (s *Service) reconcileClassicLoadBalancer() error {
 			}
 		}
 
+		// BUG: note that describeClassicELB doesn't set HealthCheck in its output,
+		// so we're configuring the health check on every reconcile whether it's
+		// needed or not.
 		if !cmp.Equal(spec.HealthCheck, apiELB.HealthCheck) {
 			s.scope.Debug("Reconciling health check for apiserver load balancer", "health-check", spec.HealthCheck)
 			err := s.configureHealthCheck(apiELB.Name, spec.HealthCheck)
@@ -597,7 +652,7 @@ func (s *Service) reconcileClassicLoadBalancer() error {
 }
 
 func (s *Service) configureHealthCheck(name string, healthCheck *infrav1.ClassicELBHealthCheck) error {
-	if _, err := s.ELBClient.ConfigureHealthCheck(&elb.ConfigureHealthCheckInput{
+	healthCheckInput := &elb.ConfigureHealthCheckInput{
 		LoadBalancerName: aws.String(name),
 		HealthCheck: &elb.HealthCheck{
 			Target:             aws.String(healthCheck.Target),
@@ -606,7 +661,14 @@ func (s *Service) configureHealthCheck(name string, healthCheck *infrav1.Classic
 			HealthyThreshold:   aws.Int64(healthCheck.HealthyThreshold),
 			UnhealthyThreshold: aws.Int64(healthCheck.UnhealthyThreshold),
 		},
-	}); err != nil {
+	}
+
+	if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) {
+		if _, err := s.ELBClient.ConfigureHealthCheck(healthCheckInput); err != nil {
+			return false, err
+		}
+		return true, nil
+	}, awserrors.LoadBalancerNotFound); err != nil {
 		return errors.Wrapf(err, "failed to configure health check for classic load balancer: %s", name)
 	}
 	return nil
@@ -1193,30 +1255,15 @@ func (s *Service) createClassicELB(spec *infrav1.LoadBalancer) (*infrav1.LoadBal
 		return nil, errors.Wrapf(err, "failed to create classic load balancer: %v", spec)
 	}
 
-	if spec.HealthCheck != nil {
-		if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) {
-			if _, err := s.ELBClient.ConfigureHealthCheck(&elb.ConfigureHealthCheckInput{
-				LoadBalancerName: aws.String(spec.Name),
-				HealthCheck: &elb.HealthCheck{
-					Target:             aws.String(spec.HealthCheck.Target),
-					Interval:           aws.Int64(int64(spec.HealthCheck.Interval.Seconds())),
-					Timeout:            aws.Int64(int64(spec.HealthCheck.Timeout.Seconds())),
-					HealthyThreshold:   aws.Int64(spec.HealthCheck.HealthyThreshold),
-					UnhealthyThreshold: aws.Int64(spec.HealthCheck.UnhealthyThreshold),
-				},
-			}); err != nil {
-				return false, err
-			}
-			return true, nil
-		}, awserrors.LoadBalancerNotFound); err != nil {
-			return nil, errors.Wrapf(err, "failed to configure health check for classic load balancer: %v", spec)
-		}
-	}
-
 	s.scope.Info("Created classic load balancer", "dns-name", *out.DNSName)
 
 	res := spec.DeepCopy()
 	res.DNSName = *out.DNSName
+
+	// We haven't configured any health check yet. Don't report it here so it
+	// will be set later during reconciliation.
+	res.HealthCheck = nil
+
 	return res, nil
 }
 



Test patch:
diff --git a/pkg/cloud/services/elb/loadbalancer_test.go b/pkg/cloud/services/elb/loadbalancer_test.go
index 073c64e0b..7a578fd7d 100644
--- a/pkg/cloud/services/elb/loadbalancer_test.go
+++ b/pkg/cloud/services/elb/loadbalancer_test.go
@@ -2546,7 +2546,10 @@ func TestReconcileV2LB(t *testing.T) {
 				scope:       clusterScope,
 				ELBV2Client: elbV2APIMocks,
 			}
-			err = s.reconcileV2LB(clusterScope.ControlPlaneLoadBalancer())
+			reconciler, err := s.getOrCreateV2LB(clusterScope.ControlPlaneLoadBalancer())
+			if err == nil {
+				err = reconciler()
+			}
 			lb := s.scope.Network().APIServerELB
 
 			tc.check(t, &lb, err)
@@ -2567,9 +2570,41 @@ func TestReconcileLoadbalancers(t *testing.T) {
 		az              = "us-west-1a"
 	)
 
+	primaryELB := func() *elbv2.LoadBalancer {
+		return &elbv2.LoadBalancer{
+			LoadBalancerArn:  aws.String(elbArn),
+			LoadBalancerName: aws.String(elbName),
+			DNSName:          aws.String(elbName),
+			Scheme:           aws.String(string(infrav1.ELBSchemeInternetFacing)),
+			AvailabilityZones: []*elbv2.AvailabilityZone{
+				{
+					SubnetId: aws.String(clusterSubnetID),
+					ZoneName: aws.String(az),
+				},
+			},
+			VpcId: aws.String(vpcID),
+		}
+	}
+
+	secondaryELB := func() *elbv2.LoadBalancer {
+		return &elbv2.LoadBalancer{
+			LoadBalancerArn:  aws.String(secondElbArn),
+			LoadBalancerName: aws.String(secondElbName),
+			DNSName:          aws.String(secondElbName),
+			Scheme:           aws.String(string(infrav1.ELBSchemeInternal)),
+			AvailabilityZones: []*elbv2.AvailabilityZone{
+				{
+					SubnetId: aws.String(clusterSubnetID),
+					ZoneName: aws.String(az),
+				},
+			},
+			VpcId: aws.String(vpcID),
+		}
+	}
+
 	tests := []struct {
 		name          string
-		elbV2APIMocks func(m *mocks.MockELBV2APIMockRecorder)
+		elbV2APIMocks func(*mocks.MockELBV2APIMockRecorder)
 		check         func(t *testing.T, firstLB, secondLB *infrav1.LoadBalancer, err error)
 		awsCluster    func(acl infrav1.AWSCluster) infrav1.AWSCluster
 		spec          func(spec infrav1.LoadBalancer) infrav1.LoadBalancer
@@ -2590,20 +2625,7 @@ func TestReconcileLoadbalancers(t *testing.T) {
 					Names: aws.StringSlice([]string{elbName}),
 				})).
 					Return(&elbv2.DescribeLoadBalancersOutput{
-						LoadBalancers: []*elbv2.LoadBalancer{
-							{
-								LoadBalancerArn:  aws.String(elbArn),
-								LoadBalancerName: aws.String(elbName),
-								Scheme:           aws.String(string(infrav1.ELBSchemeInternetFacing)),
-								AvailabilityZones: []*elbv2.AvailabilityZone{
-									{
-										SubnetId: aws.String(clusterSubnetID),
-										ZoneName: aws.String(az),
-									},
-								},
-								VpcId: aws.String(vpcID),
-							},
-						},
+						LoadBalancers: []*elbv2.LoadBalancer{primaryELB()},
 					}, nil)
 				m.DescribeLoadBalancerAttributes(&elbv2.DescribeLoadBalancerAttributesInput{LoadBalancerArn: aws.String(elbArn)}).Return(
 					&elbv2.DescribeLoadBalancerAttributesOutput{
@@ -2632,20 +2654,7 @@ func TestReconcileLoadbalancers(t *testing.T) {
 					Names: aws.StringSlice([]string{secondElbName}),
 				})).
 					Return(&elbv2.DescribeLoadBalancersOutput{
-						LoadBalancers: []*elbv2.LoadBalancer{
-							{
-								LoadBalancerArn:  aws.String(secondElbArn),
-								LoadBalancerName: aws.String(secondElbName),
-								Scheme:           aws.String(string(infrav1.ELBSchemeInternal)),
-								AvailabilityZones: []*elbv2.AvailabilityZone{
-									{
-										SubnetId: aws.String(clusterSubnetID),
-										ZoneName: aws.String(az),
-									},
-								},
-								VpcId: aws.String(vpcID),
-							},
-						},
+						LoadBalancers: []*elbv2.LoadBalancer{secondaryELB()},
 					}, nil)
 				m.DescribeLoadBalancerAttributes(&elbv2.DescribeLoadBalancerAttributesInput{LoadBalancerArn: aws.String(secondElbArn)}).Return(
 					&elbv2.DescribeLoadBalancerAttributesOutput{
@@ -2690,6 +2699,121 @@ func TestReconcileLoadbalancers(t *testing.T) {
 				}
 			},
 		},
+		{
+			name: "ensure two load balancers are created concurrently",
+			awsCluster: func(acl infrav1.AWSCluster) infrav1.AWSCluster {
+				acl.Spec.ControlPlaneLoadBalancer.Name = aws.String(elbName)
+				acl.Spec.SecondaryControlPlaneLoadBalancer = &infrav1.AWSLoadBalancerSpec{
+					Name:             aws.String(secondElbName),
+					Scheme:           &infrav1.ELBSchemeInternal,
+					LoadBalancerType: infrav1.LoadBalancerTypeNLB,
+				}
+				return acl
+			},
+			elbV2APIMocks: func(m *mocks.MockELBV2APIMockRecorder) {
+				// Initial DescribeLoadBalancers return empty
+				m.DescribeLoadBalancers(gomock.Eq(&elbv2.DescribeLoadBalancersInput{
+					Names: aws.StringSlice([]string{elbName}),
+				})).Return(&elbv2.DescribeLoadBalancersOutput{}, nil)
+				m.DescribeLoadBalancers(gomock.Eq(&elbv2.DescribeLoadBalancersInput{
+					Names: aws.StringSlice([]string{secondElbName}),
+				})).Return(&elbv2.DescribeLoadBalancersOutput{}, nil)
+
+				// Create both load balancers
+				createLB1 := m.CreateLoadBalancer(gomock.Eq(&elbv2.CreateLoadBalancerInput{
+					Name:           aws.String(elbName),
+					Scheme:         aws.String(string(infrav1.ELBSchemeInternetFacing)),
+					SecurityGroups: []*string{aws.String("")},
+					Subnets:        []*string{},
+					Tags: []*elbv2.Tag{
+						{
+							Key:   aws.String("Name"),
+							Value: aws.String(elbName),
+						},
+						{
+							Key:   aws.String(infrav1.ClusterTagKey(clusterName)),
+							Value: aws.String(string(infrav1.ResourceLifecycleOwned)),
+						},
+						{
+							Key:   aws.String(infrav1.NameAWSClusterAPIRole),
+							Value: aws.String(infrav1.APIServerRoleTagValue),
+						},
+					},
+					Type: aws.String("network"),
+				})).Return(&elbv2.CreateLoadBalancerOutput{
+					LoadBalancers: []*elbv2.LoadBalancer{primaryELB()},
+				}, nil)
+
+				createLB2 := m.CreateLoadBalancer(gomock.Eq(&elbv2.CreateLoadBalancerInput{
+					Name:           aws.String(secondElbName),
+					Scheme:         aws.String(string(infrav1.ELBSchemeInternal)),
+					SecurityGroups: []*string{aws.String("")},
+					Subnets:        []*string{},
+					Tags: []*elbv2.Tag{
+						{
+							Key:   aws.String("Name"),
+							Value: aws.String(secondElbName),
+						},
+						{
+							Key:   aws.String(infrav1.ClusterTagKey(clusterName)),
+							Value: aws.String(string(infrav1.ResourceLifecycleOwned)),
+						},
+						{
+							Key:   aws.String(infrav1.NameAWSClusterAPIRole),
+							Value: aws.String(infrav1.APIServerRoleTagValue),
+						},
+					},
+					Type: aws.String("network"),
+				})).Return(&elbv2.CreateLoadBalancerOutput{
+					LoadBalancers: []*elbv2.LoadBalancer{secondaryELB()},
+				}, nil)
+
+				// Assert that we don't wait for either load balancer to be
+				// available until both load balancers have been created
+				m.WaitUntilLoadBalancerAvailableWithContext(gomock.Any(), gomock.Eq(&elbv2.DescribeLoadBalancersInput{
+					LoadBalancerArns: aws.StringSlice([]string{elbArn}),
+				})).Return(nil).After(createLB1).After(createLB2)
+				m.WaitUntilLoadBalancerAvailableWithContext(gomock.Any(), gomock.Eq(&elbv2.DescribeLoadBalancersInput{
+					LoadBalancerArns: aws.StringSlice([]string{secondElbArn}),
+				})).Return(nil).After(createLB1).After(createLB2)
+
+				// Make minimal assertions on other calls not under test
+				m.DescribeTargetGroups(gomock.Any()).Return(&elbv2.DescribeTargetGroupsOutput{}, nil).AnyTimes()
+				m.DescribeListeners(gomock.Any()).Return(&elbv2.DescribeListenersOutput{}, nil).AnyTimes()
+				m.ModifyTargetGroupAttributes(gomock.Any()).Return(nil, nil).AnyTimes()
+				m.ModifyListener(gomock.Any()).Return(nil, nil).AnyTimes()
+				m.SetSecurityGroups(gomock.Any()).Return(nil, nil).AnyTimes()
+
+				// These calls return the same info for both load balancers, but it's not important to this test
+				m.CreateTargetGroup(gomock.Any()).Return(&elbv2.CreateTargetGroupOutput{
+					TargetGroups: []*elbv2.TargetGroup{
+						{
+							TargetGroupName: aws.String(elbName),
+							TargetGroupArn:  aws.String(elbArn),
+						},
+					},
+				}, nil).AnyTimes()
+				m.CreateListener(gomock.Any()).Return(&elbv2.CreateListenerOutput{
+					Listeners: []*elbv2.Listener{
+						{
+							ListenerArn: aws.String(elbArn),
+						},
+					},
+				}, nil).AnyTimes()
+			},
+			check: func(t *testing.T, firstLB, secondLB *infrav1.LoadBalancer, err error) {
+				t.Helper()
+				if err != nil {
+					t.Fatalf("did not expect error: %v", err)
+				}
+				if firstLB == nil {
+					t.Errorf("Expected first LB to be populated, was nil")
+				}
+				if secondLB == nil {
+					t.Errorf("Expected second LB to be populated, was nil")
+				}
+			},
+		},
 	}
 
 	for _, tc := range tests {