Repository: fedora-infra/bodhi

Issue:
Ensure the masher's Koji interactions are fault-tolerant
This is an expansion on https://github.com/fedora-infra/bodhi/issues/1143.

The masher is a long-running task that needs to handle failures gracefully. The masher's interactions with Koji need to work despite intermittent network problems. This includes authentication, which involves the Kerberos server.

- [x] Ensure calls to Koji are retried in the event of network problems

- [ ]  Ensure that if there is an authentication problem, it is retried


Discussion:
I think we can use ```backoff``` in ```bodhi.server.buildsys.koji_login()``` to help with Koji auth errors. I get tracebacks like this one from time to time, and it would be wonderful to eliminate them:

```
2019-06-07 00:13:24,087 ERROR [bodhi][f29-updates] ComposerThread failed. Transaction rolled back.
Traceback (most recent call last):
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/composer.py", line 316, in run
    self.work()
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/composer.py", line 386, in work
    self._compose_updates()
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/composer.py", line 923, in _compose_updates
    uinfo = self._generate_updateinfo()
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/composer.py", line 992, in _generate_updateinfo
    self.db, self.compose_dir)
  File "/usr/lib/python3.7/site-packages/bodhi/server/metadata.py", line 148, in __init__
    self.add_update(update)
  File "/usr/lib/python3.7/site-packages/bodhi/server/metadata.py", line 234, in add_update
    koji = get_session()
  File "/usr/lib/python3.7/site-packages/bodhi/server/buildsys.py", line 578, in get_session
    return _buildsystem()
  File "/usr/lib/python3.7/site-packages/bodhi/server/buildsys.py", line 610, in get_koji_login
    return koji_login(config=settings, authenticate=authenticate)
  File "/usr/lib/python3.7/site-packages/bodhi/server/buildsys.py", line 535, in koji_login
    if authenticate and not koji_client.krb_login(**get_krb_conf(config)):
  File "/usr/lib/python3.7/site-packages/koji/__init__.py", line 2161, in krb_login
    if self.gssapi_login(principal, keytab, ccache, proxyuser=proxyuser):
  File "/usr/lib/python3.7/site-packages/koji/__init__.py", line 2315, in gssapi_login
    raise AuthError('unable to obtain a session')
koji.AuthError: unable to obtain a session
```

Resolving patch:
diff --git a/bodhi/server/buildsys.py b/bodhi/server/buildsys.py
--- a/bodhi/server/buildsys.py
+++ b/bodhi/server/buildsys.py
@@ -24,6 +24,7 @@
 import os
 from functools import wraps
 
+import backoff
 import koji
 
 if typing.TYPE_CHECKING:  # pragma: no cover
@@ -525,6 +526,7 @@ def getRPMHeaders(self, rpmID: str,
             return headers
 
 
+@backoff.on_exception(backoff.expo, koji.AuthError, max_time=600)
 def koji_login(config: 'BodhiConfig', authenticate: bool) -> koji.ClientSession:
     """
     Login to Koji and return the session.


Test patch:
diff --git a/bodhi/tests/server/test_buildsys.py b/bodhi/tests/server/test_buildsys.py
--- a/bodhi/tests/server/test_buildsys.py
+++ b/bodhi/tests/server/test_buildsys.py
@@ -105,6 +105,40 @@ def test_krb_principal(self):
 class TestKojiLogin(unittest.TestCase):
     """This class contains tests for the koji_login() function."""
 
+    @mock.patch.object(buildsys, '_koji_hub', 'http://example.com/koji')
+    @mock.patch('bodhi.server.buildsys.koji.ClientSession.krb_login')
+    @mock.patch('bodhi.server.buildsys.log.error')
+    @mock.patch('time.sleep')
+    def test_AuthError(self, sleep, error, krb_login):
+        """backoff should take effect if an AuthError is raised."""
+        krb_login.side_effect = [koji.AuthError, koji.AuthError, True]
+        config = {'some_meaningless_other_key': 'boring_value', 'krb_ccache': 'a_ccache',
+                  'krb_keytab': 'a_keytab', 'krb_principal': 'a_principal'}
+        default_koji_opts = {
+            'krb_rdns': False,
+            'max_retries': 30,
+            'retry_interval': 10,
+            'offline_retry': True,
+            'offline_retry_interval': 10,
+            'anon_retry': True,
+        }
+
+        client = buildsys.koji_login(config, authenticate=True)
+
+        for key in default_koji_opts:
+            self.assertEqual(default_koji_opts[key], client.opts[key])
+        self.assertEqual(type(client), koji.ClientSession)
+        # Due to the use of backoff, we should have called krb_login three times.
+        self.assertEqual(
+            krb_login.mock_calls,
+            [mock.call(ccache='a_ccache', keytab='a_keytab', principal='a_principal')] * 3)
+        # No error should have been logged
+        self.assertEqual(error.call_count, 0)
+        # Make sure sleep was called twice, but we don't want to assert what values were used
+        # because that's up to backoff and we don't want different versions of backoff to make
+        # different choices and cause this test to fail.
+        self.assertEqual(sleep.call_count, 2)
+
     @mock.patch.object(buildsys, '_koji_hub', 'http://example.com/koji')
     @mock.patch('bodhi.server.buildsys.koji.ClientSession.krb_login')
     @mock.patch('bodhi.server.buildsys.log.error')