Repository: fedora-infra/bodhi

Issue:
ConnectionResetError in _wait_for_sync()
# Workaround

This exception is raised in the code that checks to see if the compose has been synchronized with the mirrors yet, so it is safe to simply resume the compose if this happens.

# The problem

Today several of the Bodhi composes failed in [```_wait_for_sync()```](https://github.com/fedora-infra/bodhi/blob/3.11.3/bodhi/server/consumers/masher.py#L1290-L1344):

```
Traceback (most recent call last):
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/masher.py", line 344, in run
    self.work()
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/masher.py", line 415, in work
    self._compose_updates()
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/masher.py", line 964, in _compose_updates
    self._wait_for_sync()
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/masher.py", line 1327, in _wait_for_sync
    masterrepomd = urllib2.urlopen(master_repomd_url)
  File "/usr/lib64/python3.7/urllib/request.py", line 222, in urlopen
    return opener.open(url, data, timeout)
  File "/usr/lib64/python3.7/urllib/request.py", line 525, in open
    response = self._open(req, data)
  File "/usr/lib64/python3.7/urllib/request.py", line 543, in _open
    '_open', req)
  File "/usr/lib64/python3.7/urllib/request.py", line 503, in _call_chain
    result = func(*args)
  File "/usr/lib64/python3.7/urllib/request.py", line 1345, in http_open
    return self.do_open(http.client.HTTPConnection, req)
  File "/usr/lib64/python3.7/urllib/request.py", line 1320, in do_open
    r = h.getresponse()
  File "/usr/lib64/python3.7/http/client.py", line 1321, in getresponse
    response.begin()
  File "/usr/lib64/python3.7/http/client.py", line 296, in begin
    version, status, reason = self._read_status()
  File "/usr/lib64/python3.7/http/client.py", line 257, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/usr/lib64/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
ConnectionResetError: [Errno 104] Connection reset by peer
```

This seems like just another new ```Exception``` type that can be raised in Python 3 that wasn't raised in Python 2, similar to https://github.com/fedora-infra/bodhi/issues/2758.


Discussion:

Resolving patch:
diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py
--- a/bodhi/server/consumers/masher.py
+++ b/bodhi/server/consumers/masher.py
@@ -1319,7 +1319,7 @@ def _wait_for_sync(self):
                 self.log.info('Polling %s' % master_repomd_url)
                 masterrepomd = urlopen(master_repomd_url)
                 newsum = hashlib.sha1(masterrepomd.read()).hexdigest()
-            except (IncompleteRead, URLError, HTTPError):
+            except (ConnectionResetError, IncompleteRead, URLError, HTTPError):
                 self.log.exception('Error fetching repomd.xml')
                 time.sleep(200)
                 continue


Test patch:
diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py
--- a/bodhi/tests/server/consumers/test_masher.py
+++ b/bodhi/tests/server/consumers/test_masher.py
@@ -3089,6 +3089,55 @@ def test_httperror(self, urlopen, sleep, publish, save):
         sleep.assert_called_once_with(200)
         save.assert_called_once_with(ComposeState.syncing_repo)
 
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_testing_master_repomd':
+            'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread.save_state')
+    @mock.patch('bodhi.server.consumers.masher.notifications.publish')
+    @mock.patch('bodhi.server.consumers.masher.time.sleep')
+    @mock.patch('bodhi.server.consumers.masher.urlopen')
+    def test_connectionreseterror(self, urlopen, sleep, publish, save):
+        """
+        Assert that an ConnectionResetError is properly caught and logged, and that the
+        algorithm continues.
+        """
+        fake_url = mock.MagicMock()
+        fake_url.read.return_value = b'---\nyaml: rules'
+        urlopen.side_effect = [ConnectionResetError(104, 'Connection reset by peer'), fake_url]
+        t = PungiComposerThread(self.semmock, self._make_msg()['body']['msg']['composes'][0],
+                                'bowlofeggs', log, self.Session, self.tempdir)
+        t.compose = self.db.query(Compose).one()
+        t.id = 'f26-updates-testing'
+        t.log = mock.MagicMock()
+        t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M"))
+        for arch in ['aarch64', 'x86_64']:
+            repodata = os.path.join(t.path, 'compose', 'Everything', arch, 'os', 'repodata')
+            os.makedirs(repodata)
+            with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd:
+                repomd.write('---\nyaml: rules')
+
+        t._wait_for_sync()
+
+        expected_calls = [
+            mock.call(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True),
+            mock.call(topic='mashtask.sync.done', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True)]
+        publish.assert_has_calls(expected_calls)
+        # Since os.listdir() isn't deterministic about the order of the items it returns, the test
+        # won't be deterministic about which of arch URL gets used. However, either one of them
+        # would be correct so we will just assert that the one that is used is used correctly.
+        arch = 'x86_64' if 'x86_64' in urlopen.mock_calls[0][1][0] else 'aarch64'
+        expected_calls = [
+            mock.call('http://example.com/pub/fedora/linux/updates/testing/17/'
+                      '{}/repodata.repomd.xml'.format(arch))
+            for i in range(2)]
+        urlopen.assert_has_calls(expected_calls)
+        t.log.exception.assert_called_once_with('Error fetching repomd.xml')
+        sleep.assert_called_once_with(200)
+        save.assert_called_once_with(ComposeState.syncing_repo)
+
     @mock.patch.dict(
         'bodhi.server.consumers.masher.config',
         {'fedora_testing_master_repomd':