Change:
return false } <ins> type MockedFakeELBV2 struct { LoadBalancers []*elbv2.LoadBalancer TargetGroups  []*elbv2.TargetGroup Listeners     []*elbv2.Listener // keys on all of these maps are ARNs LoadBalancerAttributes map[string]map[string]string Tags                   map[string][]elbv2.Tag RegisteredInstances    map[string][]string // value is list of instance IDs } func (m *MockedFakeELBV2) AddTags(request *elbv2.AddTagsInput) (*elbv2.AddTagsOutput, error) { for _, arn := range request.ResourceArns { for _, tag := range request.Tags { m.Tags[aws.StringValue(arn)] = append(m.Tags[aws.StringValue(arn)], *tag) } } return &elbv2.AddTagsOutput{}, nil } func (m *MockedFakeELBV2) CreateLoadBalancer(request *elbv2.CreateLoadBalancerInput) (*elbv2.CreateLoadBalancerOutput, error) { accountID := 123456789 arn := fmt.Sprintf("arn:aws:elasticloadbalancing:us-east-1:%d:loadbalancer/net/%x/%x", accountID, rand.Uint64(), rand.Uint32()) newLB := &elbv2.LoadBalancer{ LoadBalancerArn:  aws.String(arn), LoadBalancerName: request.Name, Type:             aws.String(elbv2.LoadBalancerTypeEnumNetwork), VpcId:            aws.String("vpc-abc123def456abc78"), } m.LoadBalancers = append(m.LoadBalancers, newLB) return &elbv2.CreateLoadBalancerOutput{ LoadBalancers: []*elbv2.LoadBalancer{newLB}, }, nil } func (m *MockedFakeELBV2) DescribeLoadBalancers(request *elbv2.DescribeLoadBalancersInput) (*elbv2.DescribeLoadBalancersOutput, error) { findMeNames := make(map[string]bool) for _, name := range request.Names { findMeNames[aws.StringValue(name)] = true } findMeARNs := make(map[string]bool) for _, arn := range request.LoadBalancerArns { findMeARNs[aws.StringValue(arn)] = true } result := []*elbv2.LoadBalancer{} for _, lb := range m.LoadBalancers { if _, present := findMeNames[aws.StringValue(lb.LoadBalancerName)]; present { result = append(result, lb) delete(findMeNames, aws.StringValue(lb.LoadBalancerName)) } else if _, present := findMeARNs[aws.StringValue(lb.LoadBalancerArn)]; present { result = append(result, lb) delete(findMeARNs, aws.StringValue(lb.LoadBalancerArn)) } } if len(findMeNames) > 0 || len(findMeARNs) > 0 { return nil, awserr.New(elbv2.ErrCodeLoadBalancerNotFoundException, "not found", nil) } return &elbv2.DescribeLoadBalancersOutput{ LoadBalancers: result, }, nil } func (m *MockedFakeELBV2) DeleteLoadBalancer(*elbv2.DeleteLoadBalancerInput) (*elbv2.DeleteLoadBalancerOutput, error) { panic("Not implemented") } func (m *MockedFakeELBV2) ModifyLoadBalancerAttributes(request *elbv2.ModifyLoadBalancerAttributesInput) (*elbv2.ModifyLoadBalancerAttributesOutput, error) { attrMap, present := m.LoadBalancerAttributes[aws.StringValue(request.LoadBalancerArn)] if !present { attrMap = make(map[string]string) m.LoadBalancerAttributes[aws.StringValue(request.LoadBalancerArn)] = attrMap } for _, attr := range request.Attributes { attrMap[aws.StringValue(attr.Key)] = aws.StringValue(attr.Value) } return &elbv2.ModifyLoadBalancerAttributesOutput{ Attributes: request.Attributes, }, nil } func (m *MockedFakeELBV2) DescribeLoadBalancerAttributes(request *elbv2.DescribeLoadBalancerAttributesInput) (*elbv2.DescribeLoadBalancerAttributesOutput, error) { attrs := []*elbv2.LoadBalancerAttribute{} if lbAttrs, present := m.LoadBalancerAttributes[aws.StringValue(request.LoadBalancerArn)]; present { for key, value := range lbAttrs { attrs = append(attrs, &elbv2.LoadBalancerAttribute{ Key:   aws.String(key), Value: aws.String(value), }) } } return &elbv2.DescribeLoadBalancerAttributesOutput{ Attributes: attrs, }, nil } func (m *MockedFakeELBV2) CreateTargetGroup(request *elbv2.CreateTargetGroupInput) (*elbv2.CreateTargetGroupOutput, error) { accountID := 123456789 arn := fmt.Sprintf("arn:aws:elasticloadbalancing:us-east-1:%d:targetgroup/%x/%x", accountID, rand.Uint64(), rand.Uint32()) newTG := &elbv2.TargetGroup{ TargetGroupArn:  aws.String(arn), TargetGroupName: request.Name, Port:            request.Port, Protocol:        request.Protocol, } m.TargetGroups = append(m.TargetGroups, newTG) return &elbv2.CreateTargetGroupOutput{ TargetGroups: []*elbv2.TargetGroup{newTG}, }, nil } func (m *MockedFakeELBV2) DescribeTargetGroups(request *elbv2.DescribeTargetGroupsInput) (*elbv2.DescribeTargetGroupsOutput, error) { var targetGroups []*elbv2.TargetGroup if request.LoadBalancerArn != nil { targetGroups = []*elbv2.TargetGroup{} for _, tg := range m.TargetGroups { for _, lbArn := range tg.LoadBalancerArns { if aws.StringValue(lbArn) == aws.StringValue(request.LoadBalancerArn) { targetGroups = append(targetGroups, tg) break } } } } else if len(request.Names) != 0 { targetGroups = []*elbv2.TargetGroup{} for _, tg := range m.TargetGroups { for _, name := range request.Names { if aws.StringValue(tg.TargetGroupName) == aws.StringValue(name) { targetGroups = append(targetGroups, tg) break } } } } else if len(request.TargetGroupArns) != 0 { targetGroups = []*elbv2.TargetGroup{} for _, tg := range m.TargetGroups { for _, arn := range request.TargetGroupArns { if aws.StringValue(tg.TargetGroupArn) == aws.StringValue(arn) { targetGroups = append(targetGroups, tg) break } } } } else { targetGroups = m.TargetGroups } return &elbv2.DescribeTargetGroupsOutput{ TargetGroups: targetGroups, }, nil } func (m *MockedFakeELBV2) ModifyTargetGroup(request *elbv2.ModifyTargetGroupInput) (*elbv2.ModifyTargetGroupOutput, error) { var matchingTargetGroup *elbv2.TargetGroup dirtyGroups := []*elbv2.TargetGroup{} for _, tg := range m.TargetGroups { if aws.StringValue(tg.TargetGroupArn) == aws.StringValue(request.TargetGroupArn) { matchingTargetGroup = tg break } } if matchingTargetGroup != nil { dirtyGroups = append(dirtyGroups, matchingTargetGroup) if request.HealthCheckEnabled != nil { matchingTargetGroup.HealthCheckEnabled = request.HealthCheckEnabled } if request.HealthCheckIntervalSeconds != nil { matchingTargetGroup.HealthCheckIntervalSeconds = request.HealthCheckIntervalSeconds } if request.HealthCheckPath != nil { matchingTargetGroup.HealthCheckPath = request.HealthCheckPath } if request.HealthCheckPort != nil { matchingTargetGroup.HealthCheckPort = request.HealthCheckPort } if request.HealthCheckProtocol != nil { matchingTargetGroup.HealthCheckProtocol = request.HealthCheckProtocol } if request.HealthCheckTimeoutSeconds != nil { matchingTargetGroup.HealthCheckTimeoutSeconds = request.HealthCheckTimeoutSeconds } if request.HealthyThresholdCount != nil { matchingTargetGroup.HealthyThresholdCount = request.HealthyThresholdCount } if request.Matcher != nil { matchingTargetGroup.Matcher = request.Matcher } if request.UnhealthyThresholdCount != nil { matchingTargetGroup.UnhealthyThresholdCount = request.UnhealthyThresholdCount } } return &elbv2.ModifyTargetGroupOutput{ TargetGroups: dirtyGroups, }, nil } func (m *MockedFakeELBV2) DeleteTargetGroup(request *elbv2.DeleteTargetGroupInput) (*elbv2.DeleteTargetGroupOutput, error) { newTargetGroups := []*elbv2.TargetGroup{} for _, tg := range m.TargetGroups { if aws.StringValue(tg.TargetGroupArn) != aws.StringValue(request.TargetGroupArn) { newTargetGroups = append(newTargetGroups, tg) } } m.TargetGroups = newTargetGroups delete(m.RegisteredInstances, aws.StringValue(request.TargetGroupArn)) return &elbv2.DeleteTargetGroupOutput{}, nil } func (m *MockedFakeELBV2) DescribeTargetHealth(request *elbv2.DescribeTargetHealthInput) (*elbv2.DescribeTargetHealthOutput, error) { healthDescriptions := []*elbv2.TargetHealthDescription{} var matchingTargetGroup *elbv2.TargetGroup for _, tg := range m.TargetGroups { if aws.StringValue(tg.TargetGroupArn) == aws.StringValue(request.TargetGroupArn) { matchingTargetGroup = tg break } } if registeredTargets, present := m.RegisteredInstances[aws.StringValue(request.TargetGroupArn)]; present { for _, target := range registeredTargets { healthDescriptions = append(healthDescriptions, &elbv2.TargetHealthDescription{ HealthCheckPort: matchingTargetGroup.HealthCheckPort, Target: &elbv2.TargetDescription{ Id:   aws.String(target), Port: matchingTargetGroup.Port, }, TargetHealth: &elbv2.TargetHealth{ State: aws.String("healthy"), }, }) } } return &elbv2.DescribeTargetHealthOutput{ TargetHealthDescriptions: healthDescriptions, }, nil } func (m *MockedFakeELBV2) DescribeTargetGroupAttributes(*elbv2.DescribeTargetGroupAttributesInput) (*elbv2.DescribeTargetGroupAttributesOutput, error) { panic("Not implemented") } func (m *MockedFakeELBV2) ModifyTargetGroupAttributes(*elbv2.ModifyTargetGroupAttributesInput) (*elbv2.ModifyTargetGroupAttributesOutput, error) { panic("Not implemented") } func (m *MockedFakeELBV2) RegisterTargets(request *elbv2.RegisterTargetsInput) (*elbv2.RegisterTargetsOutput, error) { arn := aws.StringValue(request.TargetGroupArn) alreadyExists := make(map[string]bool) for _, targetID := range m.RegisteredInstances[arn] { alreadyExists[targetID] = true } for _, target := range request.Targets { if !alreadyExists[aws.StringValue(target.Id)] { m.RegisteredInstances[arn] = append(m.RegisteredInstances[arn], aws.StringValue(target.Id)) } } return &elbv2.RegisterTargetsOutput{}, nil } func (m *MockedFakeELBV2) DeregisterTargets(request *elbv2.DeregisterTargetsInput) (*elbv2.DeregisterTargetsOutput, error) { removeMe := make(map[string]bool) for _, target := range request.Targets { removeMe[aws.StringValue(target.Id)] = true } newRegisteredInstancesForArn := []string{} for _, targetID := range m.RegisteredInstances[aws.StringValue(request.TargetGroupArn)] { if !removeMe[targetID] { newRegisteredInstancesForArn = append(newRegisteredInstancesForArn, targetID) } } m.RegisteredInstances[aws.StringValue(request.TargetGroupArn)] = newRegisteredInstancesForArn return &elbv2.DeregisterTargetsOutput{}, nil } func (m *MockedFakeELBV2) CreateListener(request *elbv2.CreateListenerInput) (*elbv2.CreateListenerOutput, error) { accountID := 123456789 arn := fmt.Sprintf("arn:aws:elasticloadbalancing:us-east-1:%d:listener/net/%x/%x/%x", accountID, rand.Uint64(), rand.Uint32(), rand.Uint32()) newListener := &elbv2.Listener{ ListenerArn:     aws.String(arn), Port:            request.Port, Protocol:        request.Protocol, DefaultActions:  request.DefaultActions, LoadBalancerArn: request.LoadBalancerArn, } m.Listeners = append(m.Listeners, newListener) for _, tg := range m.TargetGroups { for _, action := range request.DefaultActions { if aws.StringValue(action.TargetGroupArn) == aws.StringValue(tg.TargetGroupArn) { tg.LoadBalancerArns = append(tg.LoadBalancerArns, request.LoadBalancerArn) break } } } return &elbv2.CreateListenerOutput{ Listeners: []*elbv2.Listener{newListener}, }, nil } func (m *MockedFakeELBV2) DescribeListeners(request *elbv2.DescribeListenersInput) (*elbv2.DescribeListenersOutput, error) { if len(request.ListenerArns) == 0 && request.LoadBalancerArn == nil { return &elbv2.DescribeListenersOutput{ Listeners: m.Listeners, }, nil } else if len(request.ListenerArns) == 0 { listeners := []*elbv2.Listener{} for _, lb := range m.Listeners { if aws.StringValue(lb.LoadBalancerArn) == aws.StringValue(request.LoadBalancerArn) { listeners = append(listeners, lb) } } return &elbv2.DescribeListenersOutput{ Listeners: listeners, }, nil } panic("Not implemented") } func (m *MockedFakeELBV2) DeleteListener(*elbv2.DeleteListenerInput) (*elbv2.DeleteListenerOutput, error) { panic("Not implemented") } func (m *MockedFakeELBV2) ModifyListener(request *elbv2.ModifyListenerInput) (*elbv2.ModifyListenerOutput, error) { modifiedListeners := []*elbv2.Listener{} for _, listener := range m.Listeners { if aws.StringValue(listener.ListenerArn) == aws.StringValue(request.ListenerArn) { if request.DefaultActions != nil { // for each old action, find the corresponding target group, and remove the listener's LB ARN from its list for _, action := range listener.DefaultActions { var targetGroupForAction *elbv2.TargetGroup for _, tg := range m.TargetGroups { if aws.StringValue(action.TargetGroupArn) == aws.StringValue(tg.TargetGroupArn) { targetGroupForAction = tg break } } if targetGroupForAction != nil { newLoadBalancerARNs := []*string{} for _, lbArn := range targetGroupForAction.LoadBalancerArns { if aws.StringValue(lbArn) != aws.StringValue(listener.LoadBalancerArn) { newLoadBalancerARNs = append(newLoadBalancerARNs, lbArn) } } targetGroupForAction.LoadBalancerArns = newLoadBalancerARNs } } listener.DefaultActions = request.DefaultActions // for each new action, add the listener's LB ARN to that action's target groups' lists for _, action := range request.DefaultActions { var targetGroupForAction *elbv2.TargetGroup for _, tg := range m.TargetGroups { if aws.StringValue(action.TargetGroupArn) == aws.StringValue(tg.TargetGroupArn) { targetGroupForAction = tg break } } if targetGroupForAction != nil { targetGroupForAction.LoadBalancerArns = append(targetGroupForAction.LoadBalancerArns, listener.LoadBalancerArn) } } } if request.Port != nil { listener.Port = request.Port } if request.Protocol != nil { listener.Protocol = request.Protocol } modifiedListeners = append(modifiedListeners, listener) } } return &elbv2.ModifyListenerOutput{ Listeners: modifiedListeners, }, nil } func (m *MockedFakeELBV2) WaitUntilLoadBalancersDeleted(*elbv2.DescribeLoadBalancersInput) error { panic("Not implemented") } func (m *MockedFakeEC2) maybeExpectDescribeSecurityGroups(clusterID, groupName string) { tags := []*ec2.Tag{ {Key: aws.String(TagNameKubernetesClusterLegacy), Value: aws.String(clusterID)}, {Key: aws.String(fmt.Sprintf("%s%s", TagNameKubernetesClusterPrefix, clusterID)), Value: aws.String(ResourceLifecycleOwned)}, } m.On("DescribeSecurityGroups", &ec2.DescribeSecurityGroupsInput{Filters: []*ec2.Filter{ newEc2Filter("group-name", groupName), newEc2Filter("vpc-id", ""), }}).Maybe().Return([]*ec2.SecurityGroup{{Tags: tags}}) m.On("DescribeSecurityGroups", &ec2.DescribeSecurityGroupsInput{}).Maybe().Return([]*ec2.SecurityGroup{{Tags: tags}}) } func TestNLBNodeRegistration(t *testing.T) { awsServices := newMockedFakeAWSServices(TestClusterID) awsServices.elbv2 = &MockedFakeELBV2{Tags: make(map[string][]elbv2.Tag), RegisteredInstances: make(map[string][]string), LoadBalancerAttributes: make(map[string]map[string]string)} c, _ := newAWSCloud(CloudConfig{}, awsServices) awsServices.ec2.(*MockedFakeEC2).Subnets = []*ec2.Subnet{ { AvailabilityZone: aws.String("us-west-2a"), SubnetId:         aws.String("subnet-abc123de"), Tags: []*ec2.Tag{ { Key:   aws.String(c.tagging.clusterTagKey()), Value: aws.String("owned"), }, }, }, } awsServices.ec2.(*MockedFakeEC2).RouteTables = []*ec2.RouteTable{ { Associations: []*ec2.RouteTableAssociation{ { Main:                    aws.Bool(true), RouteTableAssociationId: aws.String("rtbassoc-abc123def456abc78"), RouteTableId:            aws.String("rtb-abc123def456abc78"), SubnetId:                aws.String("subnet-abc123de"), }, }, RouteTableId: aws.String("rtb-abc123def456abc78"), Routes: []*ec2.Route{ { DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId:            aws.String("igw-abc123def456abc78"), State:                aws.String("active"), }, }, }, } awsServices.ec2.(*MockedFakeEC2).maybeExpectDescribeSecurityGroups(TestClusterID, "k8s-elb-aid") nodes := []*v1.Node{makeNamedNode(awsServices, 0, "a"), makeNamedNode(awsServices, 1, "b"), makeNamedNode(awsServices, 2, "c")} fauxService := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "myservice", UID:  "id", Annotations: map[string]string{ "service.beta.kubernetes.io/aws-load-balancer-type": "nlb", }, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ { Name:       "http", Port:       8080, NodePort:   31173, TargetPort: intstr.FromInt(31173), Protocol:   v1.ProtocolTCP, }, }, SessionAffinity: v1.ServiceAffinityNone, }, } _, err := c.EnsureLoadBalancer(context.TODO(), TestClusterName, fauxService, nodes) if err != nil { t.Errorf("EnsureLoadBalancer returned an error: %v", err) } for _, instances := range awsServices.elbv2.(*MockedFakeELBV2).RegisteredInstances { if len(instances) != 3 { t.Errorf("Expected 3 nodes registered with target group, saw %d", len(instances)) } } _, err = c.EnsureLoadBalancer(context.TODO(), TestClusterName, fauxService, nodes[:2]) if err != nil { t.Errorf("EnsureLoadBalancer returned an error: %v", err) } for _, instances := range awsServices.elbv2.(*MockedFakeELBV2).RegisteredInstances { if len(instances) != 2 { t.Errorf("Expected 2 nodes registered with target group, saw %d", len(instances)) } } _, err = c.EnsureLoadBalancer(context.TODO(), TestClusterName, fauxService, nodes) if err != nil { t.Errorf("EnsureLoadBalancer returned an error: %v", err) } for _, instances := range awsServices.elbv2.(*MockedFakeELBV2).RegisteredInstances { if len(instances) != 3 { t.Errorf("Expected 3 nodes registered with target group, saw %d", len(instances)) } } } func makeNamedNode(s *FakeAWSServices, offset int, name string) *v1.Node { instanceID := fmt.Sprintf("i-%x", int64(0x02bce90670bb0c7cd)+int64(offset)) instance := &ec2.Instance{} instance.InstanceId = aws.String(instanceID) instance.Placement = &ec2.Placement{ AvailabilityZone: aws.String("us-east-1c"), } instance.PrivateDnsName = aws.String(fmt.Sprintf("ip-172-20-0-%d.ec2.internal", 101+offset)) instance.PrivateIpAddress = aws.String(fmt.Sprintf("192.168.0.%d", 1+offset)) var tag ec2.Tag tag.Key = aws.String(TagNameKubernetesClusterLegacy) tag.Value = aws.String(TestClusterID) instance.Tags = []*ec2.Tag{&tag} s.instances = append(s.instances, instance) testProviderID := "aws:///us-east-1c/" + instanceID return &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: v1.NodeSpec{ ProviderID: testProviderID, }, } } </ins> func newMockedFakeAWSServices(id string) *FakeAWSServices { s := NewFakeAWSServices(id) s.ec2 = &MockedFakeEC2{FakeEC2Impl: s.ec2.(*FakeEC2Impl)}

Discussion:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Using a pod like the nginx-ing-cntlr with an AWS Network Load Balancer (NLB), and then draining the node its running on to force a migration to another node pool, does not reliably deregister the drained nodes from the NLB Target group. This causes the nodeport used by the NLB to remain open, and the drained node ends up continuing to funnel traffic for other pods cause of being used. This leads to connection timeouts for clients, and downtime on migrations. Switching to a Classic ELB, instead of an NLB, with connection draining enabled does work, and the instances that are drained are practically removed immediately from the instances targeted by the ELB, as expected. There seems to be some sort of issue with NLB's as this is not happening, or happening rarely (only saw it once in many tests but couldn't recreate it). The node + LB set seems to be working as I see in the controller-mgr logs that the nodes are updated as I went from 11 nodes to 8, after attempting to drain + migrate a nodepool of 3 nodes with nginx-ing-cntlr all running on them: However, this update does not seem to be translating into deregistering instances from the NLB. And i've waited minutes & hours in case timeouts were playing a hand, but that didn't seem to help. What you expected to happen: On node draining, with being used, that the NLB would deregister the instances as expected. How to reproduce it (as minimally and precisely as possible): Launch a Deployment + NLB service on a node and drain it to force it over to another node pool. Watch the instances in the AWS NLB Target group never get out of "healthy" status or be drained + removed. Anything else we need to know?: Environment: AWS EKS Kubernetes version (use ): 1.12 & 1.13 Cloud provider or hardware configuration: AWS OS (e.g: ): AWS Linux 2 Kernel (e.g.

): 4.14.123-111.109.amzn2.x8664 Install tools: Network plugin and version (if this is a network-related bug): core-dns v1.2.6 Others:
/sig aws
Related:
/sig cloud-provider
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details