Repository: fedora-infra/bodhi

Issue:
IncompleteRead while Composer polls for repomd files
It seems that Python's http library can raise a new Exception in Python 3 that we don't catch:

```
[2018-11-20 14:42:34][fedmsg.consumers    INFO]epel7-testing Polling http://download01.phx2.fedoraproject.org/pub/epel/testing/7/aarch64/repodata/repomd.xml
[2018-11-20 14:42:34][fedmsg.consumers   ERROR]epel7-testing Exception in ComposerThread(epel7-testing)
Traceback (most recent call last):
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/masher.py", line 410, in work
    self._compose_updates()
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/masher.py", line 959, in _compose_updates
    self._wait_for_sync()
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/masher.py", line 1327, in _wait_for_sync
    newsum = hashlib.sha1(masterrepomd.read()).hexdigest()
  File "/usr/lib64/python3.7/http/client.py", line 460, in read
    s = self._safe_read(self.length)
  File "/usr/lib64/python3.7/http/client.py", line 612, in _safe_read
    raise IncompleteRead(b''.join(s), amt)
http.client.IncompleteRead: IncompleteRead(3295 bytes read, 1 more expected)
```

This one should be pretty easy to fix - we just need to catch this new Exception and retry, similar to what we do if a URLError is raised.


Discussion:

Resolving patch:
diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py
--- a/bodhi/server/consumers/masher.py
+++ b/bodhi/server/consumers/masher.py
@@ -34,6 +34,11 @@
 import threading
 import time
 from datetime import datetime
+try:
+    from http.client import IncompleteRead
+except ImportError:  # pragma: no cover
+    # Python 2 does not have this Exception.
+    IncompleteRead = None
 
 import fedmsg.consumers
 import jinja2
@@ -1320,7 +1325,7 @@ def _wait_for_sync(self):
             try:
                 self.log.info('Polling %s' % master_repomd_url)
                 masterrepomd = urllib2.urlopen(master_repomd_url)
-            except (URLError, HTTPError):
+            except (IncompleteRead, URLError, HTTPError):
                 self.log.exception('Error fetching repomd.xml')
                 time.sleep(200)
                 continue


Test patch:
diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py
--- a/bodhi/tests/server/consumers/test_masher.py
+++ b/bodhi/tests/server/consumers/test_masher.py
@@ -25,6 +25,11 @@
 import tempfile
 import time
 import unittest
+try:
+    from http.client import IncompleteRead
+except ImportError:
+    # Python 2 does not have this Exception.
+    IncompleteRead = None
 
 from click import testing
 import mock
@@ -3088,6 +3093,55 @@ def test_httperror(self, urlopen, sleep, publish, save):
         sleep.assert_called_once_with(200)
         save.assert_called_once_with(ComposeState.syncing_repo)
 
+    @unittest.skipIf(six.PY2, "Python 2 does not have the IncompleteRead Exception.")
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_testing_master_repomd':
+            'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread.save_state')
+    @mock.patch('bodhi.server.consumers.masher.notifications.publish')
+    @mock.patch('bodhi.server.consumers.masher.time.sleep')
+    @mock.patch('bodhi.server.consumers.masher.urllib2.urlopen')
+    def test_incompleteread(self, urlopen, sleep, publish, save):
+        """
+        Assert that an IncompleteRead is properly caught and logged, and that the code continues.
+        """
+        fake_url = mock.MagicMock()
+        fake_url.read.return_value = b'---\nyaml: rules'
+        urlopen.side_effect = [IncompleteRead('some_data'), fake_url]
+        t = PungiComposerThread(self.semmock, self._make_msg()['body']['msg']['composes'][0],
+                                'bowlofeggs', log, self.Session, self.tempdir)
+        t.compose = self.db.query(Compose).one()
+        t.id = 'f26-updates-testing'
+        t.log = mock.MagicMock()
+        t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M"))
+        for arch in ['aarch64', 'x86_64']:
+            repodata = os.path.join(t.path, 'compose', 'Everything', arch, 'os', 'repodata')
+            os.makedirs(repodata)
+            with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd:
+                repomd.write('---\nyaml: rules')
+
+        t._wait_for_sync()
+
+        expected_calls = [
+            mock.call(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True),
+            mock.call(topic='mashtask.sync.done', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True)]
+        publish.assert_has_calls(expected_calls)
+        # Since os.listdir() isn't deterministic about the order of the items it returns, the test
+        # won't be deterministic about which of arch URL gets used. However, either one of them
+        # would be correct so we will just assert that the one that is used is used correctly.
+        arch = 'x86_64' if 'x86_64' in urlopen.mock_calls[0][1][0] else 'aarch64'
+        expected_calls = [
+            mock.call('http://example.com/pub/fedora/linux/updates/testing/17/'
+                      '{}/repodata.repomd.xml'.format(arch))
+            for i in range(2)]
+        urlopen.assert_has_calls(expected_calls)
+        t.log.exception.assert_called_once_with('Error fetching repomd.xml')
+        sleep.assert_called_once_with(200)
+        save.assert_called_once_with(ComposeState.syncing_repo)
+
     @mock.patch.dict(
         'bodhi.server.consumers.masher.config',
         {'fedora_testing_master_repomd': None})