Repository: fedora-infra/bodhi

Issue:
Create a fedmsg transaction manager
It would be awesome to hook in our fedmsg notification shim to the repoze.tm2 transaction middleware. Right now the consumers on the bodhi backend are getting fedmsgs before the frontend commits them to the database. If we create this layer, then fedmsgs will get sent only when `transaction.commit()` is called.

https://zodb.readthedocs.org/en/latest/transactions.html#the-pickledatamanager



Discussion:

Resolving patch:
diff --git a/bodhi/__init__.py b/bodhi/__init__.py
--- a/bodhi/__init__.py
+++ b/bodhi/__init__.py
@@ -192,6 +192,7 @@ def main(global_config, testing=None, **settings):
     config.scan('bodhi.views')
     config.scan('bodhi.services')
     config.scan('bodhi.captcha')
+    config.scan('bodhi.events')
 
     # Setup fedmsg for this thread
     import bodhi.notifications
diff --git a/bodhi/events.py b/bodhi/events.py
new file mode 100644
--- /dev/null
+++ b/bodhi/events.py
@@ -0,0 +1,26 @@
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+from pyramid.events import NewRequest
+from pyramid.events import subscriber
+
+from bodhi import log
+import bodhi.notifications
+
+## Optionally use this to debug the number of transaction/data managers in the
+## _managers_map.  Things look fine here, but we could enable this later if we
+## want to inspect stuff.
+#@subscriber(NewRequest)
+#def fedmsg_manager_debugger(event):
+#    log.debug("fedmsg_manager_debugger %r" % bodhi.notifications._managers_map)
diff --git a/bodhi/notifications.py b/bodhi/notifications.py
--- a/bodhi/notifications.py
+++ b/bodhi/notifications.py
@@ -12,6 +12,10 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
+import copy
+
+import transaction
+
 import fedmsg
 import fedmsg.config
 
@@ -42,5 +46,112 @@ def publish(topic, msg):
         bodhi.log.warn("fedmsg disabled.  not sending %r" % topic)
         return
 
-    bodhi.log.debug("fedmsg sending %r" % topic)
-    fedmsg.publish(topic=topic, msg=msg)
+    bodhi.log.debug("fedmsg enqueueing %r" % topic)
+
+    manager = _managers_map.get_current_data_manager()
+    manager.enqueue(topic, msg)
+
+
+class ManagerMapping(object):
+    """ Maintain a two-way one-to-one mapping between transaction managers and
+    data managers (for different wsgi threads in the same process). """
+
+    def __init__(self):
+        self._left = {}
+        self._right = {}
+
+    def get_current_data_manager(self):
+        current_transaction_manager = transaction.get()
+        if not current_transaction_manager in self:
+            current_data_manager = FedmsgDataManager()
+            self.add(current_transaction_manager, current_data_manager)
+            current_transaction_manager.join(current_data_manager)
+        else:
+            current_data_manager = self.get(current_transaction_manager)
+        return current_data_manager
+
+    def add(self, transaction_manager, data_manager):
+        self._left[transaction_manager] = data_manager
+        self._right[data_manager] = transaction_manager
+
+    def __contains__(self, item):
+        return item in self._left or item in self._right
+
+    def get(self, transaction_manager):
+        return self._left[transaction_manager]
+
+    def remove(self, data_manager):
+        transaction_manager = self._right[data_manager]
+        del self._left[transaction_manager]
+        del self._right[data_manager]
+
+    def __repr__(self):
+        return "<ManagerMapping: left(%i) right(%i)>" % (
+            len(self._left),
+            len(self._right),
+        )
+
+
+# This is a global object we'll maintain to keep track of the relationship
+# between transaction managers and our data managers.  It ensures that we don't
+# create multiple data managers per transaction and that we don't join the same
+# data manager to a transaction multiple times.  Our data manager should clean
+# up after itself and remove old tm/dm pairs from this mapping in the event of
+# abort or commit.
+_managers_map = ManagerMapping()
+
+
+class FedmsgDataManager(object):
+    transaction_manager = transaction.manager
+
+    def __init__(self):
+        self.uncommitted = []
+        self.committed = []
+
+    def enqueue(self, topic, msg):
+        self.uncommitted.append((topic, msg,))
+
+    def __repr__(self):
+        return self.uncommitted.__repr__()
+
+    def abort(self, transaction):
+        self.uncommitted = copy.copy(self.committed)
+        _managers_map.remove(self)
+
+    def tpc_begin(self, transaction):
+        pass
+
+    def commit(self, transaction):
+        pass
+
+    def tpc_vote(self, transaction):
+        #raise NotImplementedError("Check that fedmsg is initialized and that messages are serializable")
+        # We're not allowed to fail after this function returns.  But we can
+        # raise an exception to cancel the transaction for the other managers.
+        pass
+
+    def tpc_abort(self, transaction):
+        return self.abort(transaction)
+
+    def tpc_finish(self, transaction):
+        for topic, msg in self.uncommitted:
+            bodhi.log.debug("fedmsg sending %r" % topic)
+            fedmsg.publish(topic=topic, msg=msg)
+        self.committed = copy.copy(self.uncommitted)
+        _managers_map.remove(self)
+
+    def sortKey(self):
+        """ Use a 'z' to make fedmsg come last, after the db is done. """
+        return 'z_fedmsgdm' + str(id(self))
+
+    def savepoint(self):
+        return FedmsgSavepoint(self)
+
+
+class FedmsgSavepoint(object):
+    def __init__(self, dm):
+        self.dm = dm
+        self.saved_committed = copy.copy(self.dm.uncommitted)
+
+    def rollback(self):
+        self.dm.uncommitted = copy.copy(self.saved_committed)


Test patch: