Repository: Miserlou/Zappa

Issue:
Certify command fails when AWS account has more than 100 hosted zones
<!--- Provide a general summary of the issue in the Title above -->
If an AWS account has more than 100 hosted zones, the certify command will fail if the domain being certified does not return in the first 100 records.

Here is the traceroute of the error that I receive:

```Traceback (most recent call last):
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/zappa/cli.py", line 2710, in handle
    sys.exit(cli.handle())
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/zappa/cli.py", line 508, in handle
    self.dispatch_command(self.command, stage)
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/zappa/cli.py", line 616, in dispatch_command
    manual=self.vargs['manual']
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/zappa/cli.py", line 1858, in certify
    self.zappa.update_route53_records(self.domain, dns_name)
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/zappa/core.py", line 2074, in update_route53_records
    hosted_zone = self.route53.get_hosted_zone(Id=zone_id)['HostedZone']['Name'][:-1]
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/botocore/client.py", line 320, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/botocore/client.py", line 596, in _make_api_call
    api_params, operation_model, context=request_context)
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/botocore/client.py", line 630, in _convert_to_request_dict
    api_params, operation_model, context)
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/botocore/client.py", line 660, in _emit_api_params
    params=api_params, model=operation_model, context=context)
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/botocore/hooks.py", line 356, in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/botocore/hooks.py", line 228, in emit
    return self._emit(event_name, kwargs)
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/botocore/hooks.py", line 211, in _emit
    response = handler(**kwargs)
  File "/Users/degroat/.virtualenvs/coderblend/lib/python2.7/site-packages/botocore/handlers.py", line 597, in fix_route53_ids
    params[name] = orig_value.split('/')[-1]
AttributeError: 'NoneType' object has no attribute 'split'

==============
```

## Possible Fix
I traced the issue back to the usage of route53.list_hosted_zones in core.py.  It only returns the first 100.  There is a list_hosted_zones_by_name method that could be used here with the param DNSName.  Another option would be simple pagination to get all of the hosted zones from the account.



Discussion:

Resolving patch:
diff --git a/zappa/core.py b/zappa/core.py
--- a/zappa/core.py
+++ b/zappa/core.py
@@ -2196,7 +2196,18 @@ def update_domain_base_path_mapping(self, domain_name, lambda_name, stage, base_
                 restApiId=api_id,
                 stage=stage
             )
+        
+    def get_all_zones(self):
+        """Same behaviour of list_host_zones, but transparently handling pagination."""
+        zones = {'HostedZones': []}
 
+        new_zones = self.route53.list_hosted_zones(MaxItems='100')
+        while new_zones['IsTruncated']:
+            zones['HostedZones'] += new_zones['HostedZones']
+            new_zones = self.route53.list_hosted_zones(Marker=new_zones['NextMarker'], MaxItems='100')
+
+        zones['HostedZones'] += new_zones['HostedZones']
+        return zones
 
     def get_domain_name(self, domain_name, route53=True):
         """
@@ -2215,7 +2226,7 @@ def get_domain_name(self, domain_name, route53=True):
             return True
 
         try:
-            zones = self.route53.list_hosted_zones()
+            zones = self.get_all_zones()
             for zone in zones['HostedZones']:
                 records = self.route53.list_resource_record_sets(HostedZoneId=zone['Id'])
                 for record in records['ResourceRecordSets']:
@@ -2817,7 +2828,7 @@ def get_hosted_zone_id_for_domain(self, domain):
         Get the Hosted Zone ID for a given domain.
 
         """
-        all_zones = self.route53.list_hosted_zones()
+        all_zones = self.get_all_zones()
         return self.get_best_match_zone(all_zones, domain)
 
     @staticmethod


Test patch:
diff --git a/tests/tests.py b/tests/tests.py
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -1463,6 +1463,7 @@ def test_get_domain_respects_route53_setting(self, client, template):
 
         # And that the route53 path still works
         zappa_core.route53.list_hosted_zones.return_value = {
+            'IsTruncated': False,
             'HostedZones': [
                 {
                     'Id': 'somezone'
@@ -1483,6 +1484,70 @@ def test_get_domain_respects_route53_setting(self, client, template):
         zappa_core.route53.list_resource_record_sets.assert_called_once_with(
             HostedZoneId='somezone')
 
+    @mock.patch('botocore.client')
+    def test_get_all_zones_normal_case(self, client):
+        zappa_core = Zappa(
+            boto_session=mock.Mock(),
+            profile_name="test",
+            aws_region="test",
+            load_credentials=False
+        )
+        zappa_core.route53 = mock.Mock()
+
+        # Check that it handle the normal case
+        zappa_core.route53.list_hosted_zones.return_value = {
+            'IsTruncated': False,
+            'HostedZones': [
+                {
+                    'Id': 'somezone'
+                }
+            ]
+        }
+
+        zones = zappa_core.get_all_zones()
+        zappa_core.route53.list_hosted_zones.assert_called_with(MaxItems='100')
+        self.assertListEqual(zones['HostedZones'], [{'Id': 'somezone'}])
+
+    @mock.patch('botocore.client')
+    def test_get_all_zones_two_pages(self, client):
+        zappa_core = Zappa(
+            boto_session=mock.Mock(),
+            profile_name="test",
+            aws_region="test",
+            load_credentials=False
+        )
+        zappa_core.route53 = mock.Mock()
+
+        # Check that it handle the normal case
+        zappa_core.route53.list_hosted_zones.side_effect = [
+            {
+                'IsTruncated': True,
+                'HostedZones': [
+                    {
+                        'Id': 'zone1'
+                    }
+                ],
+                'NextMarker': "101"
+            },
+            {
+                'IsTruncated': False,
+                'HostedZones': [
+                    {
+                        'Id': 'zone2'
+                    }
+                ]
+            }
+        ]
+
+        zones = zappa_core.get_all_zones()
+        zappa_core.route53.list_hosted_zones.assert_has_calls(
+            [
+                mock.call(MaxItems='100'),
+                mock.call(MaxItems='100', Marker='101'),
+            ]
+        )
+        self.assertListEqual(zones['HostedZones'], [{'Id': 'zone1'}, {'Id': 'zone2'}])
+
     ##
     # Django
     ##