Repository: zalando-stups/senza

Issue:
senza traffic displays 0.0% weight for all stacks
When executing `senza traffic [name]` the results displays a weight of 0.0% for all stacks.
`senza domains` shows the weight correctly.

Discussion:

Patch:
diff --git a/senza/manaus/route53.py b/senza/manaus/route53.py
index 6aab215..38c177e 100644
--- a/senza/manaus/route53.py
+++ b/senza/manaus/route53.py
@@ -349,9 +349,22 @@ class Route53:
         if name is not None and not name.endswith('.'):
             name += '.'
         for zone in cls.get_hosted_zones():
-            # TODO use paginator
             response = client.list_resource_record_sets(HostedZoneId=zone.id)
             resources = response["ResourceRecordSets"]  # type: List[Dict[str, Any]]
+
+            # If the response includes more than maxitems resource record sets,
+            # the value of the IsTruncated element in the response is true,
+            # and the values of the NextRecordName and NextRecordType elements
+            # in the response identify the first resource record set in the
+            # next group of maxitems resource record sets.
+            while response.get('IsTruncated', False):
+                next_name = response['NextRecordName']
+                next_type = response['NextRecordType']
+                response = client.list_resource_record_sets(HostedZoneId=zone.id,
+                                                            StartRecordName=next_name,
+                                                            StartRecordType=next_type)
+                resources.extend(response['ResourceRecordSets'])
+
             for resource in resources:
                 record = Route53Record.from_boto_dict(resource,
                                                       hosted_zone=zone)


Test patch:
diff --git a/tests/test_components.py b/tests/test_components.py
index 651b318..e7207a3 100644
--- a/tests/test_components.py
+++ b/tests/test_components.py
@@ -28,7 +28,8 @@ from senza.components.weighted_dns_elastic_load_balancer import \
 from senza.components.weighted_dns_elastic_load_balancer_v2 import \
     component_weighted_dns_elastic_load_balancer_v2
 
-from fixtures import HOSTED_ZONE_ZO_NE_COM, HOSTED_ZONE_ZO_NE_DEV, boto_resource
+from fixtures import (HOSTED_ZONE_ZO_NE_COM, HOSTED_ZONE_ZO_NE_DEV,  # noqa: F401
+                      boto_resource, boto_client)
 
 
 def test_invalid_component():
@@ -180,7 +181,7 @@ def test_component_load_balancer_http_only(monkeypatch):
         "Name": "test_lb",
         "SecurityGroups": "",
         "HTTPPort": "9999",
-        "SSLCertificateId": "arn:none", # should be ignored as we overwrite Listeners
+        "SSLCertificateId": "arn:none",  # should be ignored as we overwrite Listeners
         "Listeners": [{"Foo": "Bar"}]
     }
     info = {'StackName': 'foobar', 'StackVersion': '0.1'}
@@ -281,9 +282,11 @@ def test_component_stups_auto_configuration_vpc_id(monkeypatch):
     sn3.tags = [{'Key': 'Name', 'Value': 'internal-3'}]
     sn3.availability_zone = 'az-1'
     ec2 = MagicMock()
+
     def get_subnets(Filters):
         assert Filters == [{'Name': 'vpc-id', 'Values': ['vpc-123']}]
         return [sn1, sn2, sn3]
+
     ec2.subnets.filter = get_subnets
     image = MagicMock()
     ec2.images.filter.return_value = [image]
@@ -350,7 +353,7 @@ def test_component_redis_cluster(monkeypatch):
     assert 'RedisReplicationGroup' in result['Resources']
     assert mock_string == result['Resources']['RedisReplicationGroup']['Properties']['SecurityGroupIds']
     assert 2 == result['Resources']['RedisReplicationGroup']['Properties']['NumCacheClusters']
-    assert True == result['Resources']['RedisReplicationGroup']['Properties']['AutomaticFailoverEnabled']
+    assert result['Resources']['RedisReplicationGroup']['Properties']['AutomaticFailoverEnabled']
     assert 'Engine' in result['Resources']['RedisReplicationGroup']['Properties']
     assert 'EngineVersion' in result['Resources']['RedisReplicationGroup']['Properties']
     assert 'CacheNodeType' in result['Resources']['RedisReplicationGroup']['Properties']
@@ -361,26 +364,15 @@ def test_component_redis_cluster(monkeypatch):
     assert 'SubnetIds' in result['Resources']['RedisSubnetGroup']['Properties']
 
 
-def test_weighted_dns_load_balancer(monkeypatch, boto_resource):
+def test_weighted_dns_load_balancer(monkeypatch, boto_client, boto_resource):  # noqa: F811
     senza.traffic.DNS_ZONE_CACHE = {}
 
-    def my_client(rtype, *args):
-        if rtype == 'route53':
-            route53 = MagicMock()
-            route53.list_hosted_zones.return_value = {'HostedZones': [HOSTED_ZONE_ZO_NE_COM],
-                                                      'IsTruncated': False,
-                                                      'MaxItems': '100'}
-            return route53
-        return MagicMock()
-
-    monkeypatch.setattr('boto3.client', my_client)
-
     configuration = {
         "Name": "test_lb",
         "SecurityGroups": "",
         "HTTPPort": "9999",
-        'MainDomain': 'great.api.zo.ne.com',
-        'VersionDomain': 'version.api.zo.ne.com'
+        'MainDomain': 'great.api.zo.ne',
+        'VersionDomain': 'version.api.zo.ne'
     }
     info = {'StackName': 'foobar', 'StackVersion': '0.1'}
     definition = {"Resources": {}}
@@ -408,20 +400,16 @@ def test_weighted_dns_load_balancer(monkeypatch, boto_resource):
     assert 'MainDomain' not in result["Resources"]["test_lb"]["Properties"]
 
 
-def test_weighted_dns_load_balancer_with_different_domains(monkeypatch, boto_resource):
+def test_weighted_dns_load_balancer_with_different_domains(monkeypatch,  # noqa: F811
+                                                           boto_client,
+                                                           boto_resource):
     senza.traffic.DNS_ZONE_CACHE = {}
 
-    def my_client(rtype, *args):
-        if rtype == 'route53':
-            route53 = MagicMock()
-            route53.list_hosted_zones.return_value = {'HostedZones': [HOSTED_ZONE_ZO_NE_DEV,
-                                                                      HOSTED_ZONE_ZO_NE_COM],
-                                                      'IsTruncated': False,
-                                                      'MaxItems': '100'}
-            return route53
-        return MagicMock()
-
-    monkeypatch.setattr('boto3.client', my_client)
+    boto_client['route53'].list_hosted_zones.return_value = {
+        'HostedZones': [HOSTED_ZONE_ZO_NE_DEV,
+                        HOSTED_ZONE_ZO_NE_COM],
+        'IsTruncated': False,
+        'MaxItems': '100'}
 
     configuration = {
         "Name": "test_lb",
@@ -589,8 +577,8 @@ def test_component_auto_scaling_group_custom_tags():
         'InstanceType': 't2.micro',
         'Image': 'foo',
         'Tags': [
-            { 'Key': 'Tag1', 'Value': 'alpha' },
-            { 'Key': 'Tag2', 'Value': 'beta' }
+            {'Key': 'Tag1', 'Value': 'alpha'},
+            {'Key': 'Tag2', 'Value': 'beta'}
         ]
     }
 
@@ -619,6 +607,7 @@ def test_component_auto_scaling_group_custom_tags():
     assert ts is not None
     assert ts["Value"] == 'FooStack-FooVersion'
 
+
 def test_component_auto_scaling_group_configurable_properties2():
     definition = {"Resources": {}}
     configuration = {
@@ -911,26 +900,15 @@ def test_get_load_balancer_name():
                            '1') == 'toolong12345678901234567890123-1'
 
 
-def test_weighted_dns_load_balancer_v2(monkeypatch, boto_resource):
+def test_weighted_dns_load_balancer_v2(monkeypatch, boto_client, boto_resource):  # noqa: F811
     senza.traffic.DNS_ZONE_CACHE = {}
 
-    def my_client(rtype, *args):
-        if rtype == 'route53':
-            route53 = MagicMock()
-            route53.list_hosted_zones.return_value = {'HostedZones': [HOSTED_ZONE_ZO_NE_COM],
-                                                      'IsTruncated': False,
-                                                      'MaxItems': '100'}
-            return route53
-        return MagicMock()
-
-    monkeypatch.setattr('boto3.client', my_client)
-
     configuration = {
         "Name": "MyLB",
         "SecurityGroups": "",
         "HTTPPort": "9999",
-        'MainDomain': 'great.api.zo.ne.com',
-        'VersionDomain': 'version.api.zo.ne.com',
+        'MainDomain': 'great.api.zo.ne',
+        'VersionDomain': 'version.api.zo.ne',
         # test overwritting specific properties in one of the resources
         'TargetGroupAttributes': [{'Key': 'deregistration_delay.timeout_seconds', 'Value': '123'}],
         # test that Security Groups are resolved
@@ -961,10 +939,18 @@ def test_weighted_dns_load_balancer_v2(monkeypatch, boto_resource):
     assert 'MyLBListener' in result["Resources"]
     assert 'MyLBTargetGroup' in result["Resources"]
 
-    assert result['Resources']['MyLBTargetGroup']['Properties']['HealthCheckPort'] == '9999'
-    assert result['Resources']['MyLBListener']['Properties']['Certificates'] == [{'CertificateArn': 'arn:aws:42'}]
+    target_group = result['Resources']['MyLBTargetGroup']
+    lb_listener = result['Resources']['MyLBListener']
+
+    assert target_group['Properties']['HealthCheckPort'] == '9999'
+    assert lb_listener['Properties']['Certificates'] == [
+        {'CertificateArn': 'arn:aws:42'}
+    ]
     # test that our custom drain setting works
-    assert result['Resources']['MyLBTargetGroup']['Properties']['TargetGroupAttributes'] == [{'Key': 'deregistration_delay.timeout_seconds', 'Value': '123'}]
+    assert target_group['Properties']['TargetGroupAttributes'] == [
+        {'Key': 'deregistration_delay.timeout_seconds',
+         'Value': '123'}
+    ]
     assert result['Resources']['MyLB']['Properties']['SecurityGroups'] == ['sg-foo']
 
 
diff --git a/tests/test_manaus/test_route53.py b/tests/test_manaus/test_route53.py
index 24c5441..658195a 100644
--- a/tests/test_manaus/test_route53.py
+++ b/tests/test_manaus/test_route53.py
@@ -164,6 +164,46 @@ def test_get_records(monkeypatch):
     assert records[0].name == 'domain.example.net.'
 
 
+def test_get_records_paginated(monkeypatch):
+    m_client = MagicMock()
+    m_client.return_value = m_client
+    hosted_zone1 = {'Config': {'PrivateZone': False},
+                    'CallerReference': '0000',
+                    'ResourceRecordSetCount': 42,
+                    'Id': '/hostedzone/random1',
+                    'Name': 'example.com.'}
+    mock_records = [{'Name': 'domain.example.com.',
+                     'ResourceRecords': [{'Value': '127.0.0.1'}],
+                     'TTL': 600,
+                     'Type': 'A'},
+                    {'Name': 'domain.example.net.',
+                     'ResourceRecords': [{'Value': '127.0.0.1'}],
+                     'TTL': 600,
+                     'Type': 'A'}
+                    ]
+    m_client.list_hosted_zones.return_value = {'MaxItems': '100',
+                                               'ResponseMetadata': {
+                                                   'HTTPStatusCode': 200,
+                                                   'RequestId': 'FakeId'
+                                               },
+                                               'HostedZones': [hosted_zone1],
+                                               'IsTruncated': False}
+
+    m_client.list_resource_record_sets.side_effect = [
+        {'ResourceRecordSets': mock_records,
+         'IsTruncated': True,
+         'NextRecordName': 'doesnt.matter.example.com',
+         'NextRecordType': 'A'},
+        {'ResourceRecordSets': mock_records,
+         'IsTruncated': False},
+    ]
+    monkeypatch.setattr('boto3.client', m_client)
+
+    route53 = Route53()
+    records = list(route53.get_records())
+    assert len(records) == 4
+
+
 def test_route53_record_boto_dict():
     record1 = Route53Record(name='test1', type='A')
     assert record1.boto_dict == {'Name': 'test1',