Repository: fedora-infra/bodhi

Issue:
Transactional failure during masher
Message
-------
[2017-04-25 02:27:57][    fedmsg   ERROR]
```python
MasherThread failed. Transaction rolled back.
```


Process Details
---------------
- host:     bodhi-backend01.phx2.fedoraproject.org
- PID:      31792
- name:     fedmsg-hub
- command:  /usr/bin/python /usr/bin/fedmsg-hub
- msg_id:   

Callstack that lead to the logging statement
--------------------------------------------
```python
  File "/usr/lib64/python2.7/threading.py", line 777 in __bootstrap
    self.__bootstrap_inner()
  File "/usr/lib64/python2.7/threading.py", line 804 in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 263 in run
    self.log.exception('MasherThread failed. Transaction rolled back.')
```

```
Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 261, in run
    self.db = None
  File "/usr/lib64/python2.7/contextlib.py", line 24, in __exit__
    self.gen.next()
  File "/usr/lib/python2.7/site-packages/bodhi/server/util.py", line 659, in __call__
    session.rollback()
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 765, in rollback
    self.transaction.rollback()
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 408, in rollback
    self._assert_active(prepared_ok=True, rollback_ok=True)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 196, in _assert_active
    "This session is in 'committed' state; no further "
InvalidRequestError: This session is in 'committed' state; no further SQL can be emitted within this transaction.
```


Discussion:
At this point, it seems likely that this is the same underlying issue as #1470, and I believe the fix I've got for it will fix this too.
My previous comment was incorrect, but @jeremycline and I figured out how to fix this and he will make it so. Thanks Jeremy!
This e-mail reveals the true problem:

Message
-------
[2017-04-27 19:54:51][bodhi.server   ERROR]
```python
An Exception was raised during a transactional session.
```


Process Details
---------------
- host:     bodhi-backend01.phx2.fedoraproject.org
- PID:      72142
- name:     fedmsg-hub
- command:  /usr/bin/python /usr/bin/fedmsg-hub
- msg_id:   

Callstack that lead to the logging statement
--------------------------------------------
```python
  File "/usr/lib64/python2.7/threading.py", line 777 in __bootstrap
    self.__bootstrap_inner()
  File "/usr/lib64/python2.7/threading.py", line 804 in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 261 in run
    self.db = None
  File "/usr/lib64/python2.7/contextlib.py", line 24 in __exit__
    self.gen.next()
  File "/usr/lib/python2.7/site-packages/bodhi/server/util.py", line 659 in __call__
    log.exception('An Exception was raised during a transactional session.')
```

```
Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/bodhi/server/util.py", line 657, in __call__
    session.commit()
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 801, in commit
    self.transaction.commit()
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 399, in commit
    self.session.dispatch.after_commit(self.session)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/event/attr.py", line 218, in __call__
    fn(*args, **kw)
  File "/usr/lib/python2.7/site-packages/bodhi/server/notifications.py", line 76, in send_fedmsgs_after_commit
    fedmsg.publish(topic=topic, msg=msg)
  File "/usr/lib/python2.7/site-packages/fedmsg/__init__.py", line 79, in _wrapper
    return func(*args, **kw)
  File "/usr/lib/python2.7/site-packages/fedmsg/__init__.py", line 94, in publish
    return __local.__context.publish(topic, msg, **kw)
  File "/usr/lib/python2.7/site-packages/fedmsg/core.py", line 306, in publish
    msg = fedmsg.crypto.sign(msg, **self.c)
  File "/usr/lib/python2.7/site-packages/fedmsg/crypto/__init__.py", line 219, in sign
    return _implementation.sign(message, **config)
  File "/usr/lib/python2.7/site-packages/fedmsg/crypto/x509.py", line 77, in sign
    digest.update(fedmsg.encoding.dumps(message))
  File "/usr/lib64/python2.7/json/encoder.py", line 209, in encode
    chunks = list(chunks)
  File "/usr/lib64/python2.7/json/encoder.py", line 434, in _iterencode
    for chunk in _iterencode_dict(o, _current_indent_level):
  File "/usr/lib64/python2.7/json/encoder.py", line 408, in _iterencode_dict
    for chunk in chunks:
  File "/usr/lib64/python2.7/json/encoder.py", line 408, in _iterencode_dict
    for chunk in chunks:
  File "/usr/lib64/python2.7/json/encoder.py", line 442, in _iterencode
    o = _default(o)
  File "/usr/lib/python2.7/site-packages/fedmsg/encoding/__init__.py", line 60, in default
    return obj.__json__()
  File "/usr/lib/python2.7/site-packages/bodhi/server/models.py", line 2075, in __json__
    request=request, anonymize=anonymize)
  File "/usr/lib/python2.7/site-packages/bodhi/server/models.py", line 197, in __json__
    return self._to_json(self, request=request, anonymize=anonymize)
  File "/usr/lib/python2.7/site-packages/bodhi/server/models.py", line 226, in _to_json
    d[attr] = cls._expand(obj, getattr(obj, attr), seen, request)
  File "/usr/lib/python2.7/site-packages/bodhi/server/models.py", line 252, in _expand
    return [cls._expand(obj, item, seen, req) for item in relation]
  File "/usr/lib/python2.7/site-packages/bodhi/server/models.py", line 254, in _expand
    return cls._to_json(relation, seen + [type(obj)], req)
  File "/usr/lib/python2.7/site-packages/bodhi/server/models.py", line 226, in _to_json
    d[attr] = cls._expand(obj, getattr(obj, attr), seen, request)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/attributes.py", line 237, in __get__
    return self.impl.get(instance_state(instance), dict_)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/attributes.py", line 583, in get
    value = self.callable_(state, passive)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/strategies.py", line 544, in _load_for_state
    return self._emit_lazyload(session, state, ident_key, passive)
  File "<string>", line 1, in <lambda>
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/strategies.py", line 614, in _emit_lazyload
    result = q.all()
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2613, in all
    return list(self)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2761, in __iter__
    return self._execute_and_instances(context)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2774, in _execute_and_instances
    close_with_result=True)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2765, in _connection_from_session
    **kw)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 893, in connection
    execution_options=execution_options)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 898, in _connection_for_bind
    engine, execution_options)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 313, in _connection_for_bind
    self._assert_active()
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 196, in _assert_active
    "This session is in 'committed' state; no further "
InvalidRequestError: This session is in 'committed' state; no further SQL can be emitted within this transaction.
```

So basically, we need to make sure we don't pass database models into fedmsg.publish, because they will get published *after* the commit, and then the objects will be stale causing this exception.

Resolving patch:
diff --git a/bodhi/server/notifications.py b/bodhi/server/notifications.py
--- a/bodhi/server/notifications.py
+++ b/bodhi/server/notifications.py
@@ -13,6 +13,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 import collections
+import json
 import logging
 import socket
 
@@ -103,12 +104,18 @@ def publish(topic, msg, force=False):
         bodhi.server.log.debug("fedmsg skipping transaction and sending %r" % topic)
         fedmsg.publish(topic=topic, msg=msg)
     else:
+        # We need to do this to ensure all the SQLAlchemy objects that could be in the messages
+        # are turned into JSON before the session is removed and expires the objects loaded with
+        # it. The JSON is decoded again because the fedmsg API doesn't state it accepts strings.
+        # An issue has been filed about this: https://github.com/fedora-infra/fedmsg/issues/407.
+        json_msg = fedmsg.encoding.dumps(msg)
+        msg_dict = json.loads(json_msg)
         # This gives us the thread-local session which we'll use to stash the fedmsg.
         # When commit is called on it, the :func:`send_fedmsgs_after_commit` is triggered.
         session = Session()
         if 'fedmsg' not in session.info:
             session.info['fedmsg'] = collections.defaultdict(list)
-        session.info['fedmsg'][topic].append(msg)
+        session.info['fedmsg'][topic].append(msg_dict)
         bodhi.server.log.debug("fedmsg enqueueing %r" % topic)
 
 


Test patch:
diff --git a/bodhi/tests/server/test_notifications.py b/bodhi/tests/server/test_notifications.py
--- a/bodhi/tests/server/test_notifications.py
+++ b/bodhi/tests/server/test_notifications.py
@@ -107,6 +107,27 @@ def test_publish(self, mock_init):
 
     @mock.patch.dict('bodhi.server.config.config', {'fedmsg_enabled': True})
     @mock.patch('bodhi.server.notifications.fedmsg_is_initialized', mock.Mock(return_value=False))
+    def test_publish_sqlalchemy_object(self, mock_init):
+        """Assert publish places the message inside the session info dict."""
+        Session.remove()
+        expected_msg = {
+            u'some_package': {
+                u'name': u'so good',
+                u'type': 0,
+                u'requirements': None,
+                u'stack': None,
+                u'stack_id': None,
+            }
+        }
+        package = models.Package(name='so good')
+        notifications.publish('demo.topic', {'some_package': package})
+        session = Session()
+        self.assertIn('fedmsg', session.info)
+        self.assertEqual(session.info['fedmsg']['demo.topic'], [expected_msg])
+        mock_init.assert_called_once_with()
+
+    @mock.patch('bodhi.server.notifications.fedmsg_is_initialized', mock.Mock(return_value=False))
+    @mock.patch.dict('bodhi.server.config.config', {'fedmsg_enabled': True})
     @mock.patch('bodhi.server.notifications.fedmsg.publish')
     def test_publish_force(self, mock_fedmsg_publish, mock_init):
         """Assert publish with the force flag sends the message immediately."""