Repository: fedora-infra/bodhi

Issue:
Update/build comps for mash
Since we don't know when koji will be able to mash, we should be fully capable of doing it ourselves in bodhi2, and that entails dealing with the comps repo.



Discussion:

Resolving patch:
diff --git a/alembic/versions/4efd62c610a6_add_stack_user_table.py b/alembic/versions/4efd62c610a6_add_stack_user_table.py
new file mode 100644
--- /dev/null
+++ b/alembic/versions/4efd62c610a6_add_stack_user_table.py
@@ -0,0 +1,34 @@
+"""Add stack_user_table and stack_group_table
+
+Revision ID: 4efd62c610a6
+Revises: d58f9439269
+Create Date: 2014-10-06 13:10:55.002150
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4efd62c610a6'
+down_revision = 'd58f9439269'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table('stack_group_table',
+        sa.Column('stack_id', sa.Integer(), nullable=True),
+        sa.Column('group_id', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['group_id'], ['groups.id'], ),
+        sa.ForeignKeyConstraint(['stack_id'], ['stacks.id'], )
+    )
+    op.create_table('stack_user_table',
+        sa.Column('stack_id', sa.Integer(), nullable=True),
+        sa.Column('user_id', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['stack_id'], ['stacks.id'], ),
+        sa.ForeignKeyConstraint(['user_id'], ['users.id'], )
+    )
+
+
+def downgrade():
+    op.drop_table('stack_user_table')
+    op.drop_table('stack_group_table')
diff --git a/bodhi/masher.py b/bodhi/masher.py
--- a/bodhi/masher.py
+++ b/bodhi/masher.py
@@ -8,15 +8,22 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+The Bodhi "Masher".
+
+This module is responsible for the process of "pushing" updates out. It's
+comprised of a fedmsg consumer that launches threads for each repository being
+mashed.
+"""
 
 import os
 import json
 import time
-import shutil
 import threading
+import subprocess
 import fedmsg.consumers
 
 from collections import defaultdict
@@ -49,23 +56,25 @@ class Masher(fedmsg.consumers.FedmsgConsumer):
     - Remove pending tags
     - request_complete
     - Send fedmsgs
+    - Update comps
+    - TODO: mash
 
-    - TODO: mash (update comps before hand?)
 Things to do while we're waiting on mash
     - Add testing updates to updates-testing digest
     - Generate/update updateinfo.xml
 
 Once mash is done:
     - inject the updateinfo it into the repodata
-
     - Sanity check the repo
     - Flip the symlinks to the new repo
-    - Generate and email stable update notices
     - Cache the new repodata
+
+    - Generate and email stable update notices
     - Wait for the repo to hit the master mirror
     - Update bugzillas
     - Add comments to updates
     - Email updates-testing digest
+
     - Unlock repo
         - unlock updates
         - see if any updates now meet the stable criteria, and set the request
@@ -194,7 +203,7 @@ def work(self):
         self.init_path()
 
         notifications.publish(topic="mashtask.mashing", msg=dict(repo=self.id,
-            updates=self.state['updates']))
+                              updates=self.state['updates']))
 
         success = False
         try:
@@ -210,6 +219,7 @@ def work(self):
                 self.save_state()
                 self.expire_buildroot_overrides()
                 self.remove_pending_tags()
+                self.update_comps()
                 self.mash()
                 self.generate_testing_digest()
                 self.complete_requests()
@@ -344,6 +354,40 @@ def remove_pending_tags(self):
         result = self.koji.multiCall()
         log.debug('result = %r' % result)
 
+    def cmd(self, cmd, cwd=None):
+        log.info('Running %r', cmd)
+        if isinstance(cmd, basestring):
+            cmd = cmd.split()
+        p = subprocess.Popen(cmd, cwd=cwd,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        if out:
+            log.debug(out)
+        if err:
+            log.error(err)
+        if p.returncode != 0:
+            log.error('return code %s', p.returncode)
+            raise Exception
+        return out, err, p.returncode
+
+    def update_comps(self):
+        """
+        Update our comps git module and merge the latest translations so we can
+        pass it to mash insert into the repodata.
+        """
+        log.info("Updating comps")
+        comps_dir = config.get('comps_dir')
+        comps_url = config.get('comps_url')
+        if not os.path.exists(comps_dir):
+            self.cmd(['git', 'clone', comps_url], os.path.dirname(comps_dir))
+        if comps_url.startswith('git://'):
+            self.cmd(['git', 'pull'], comps_dir)
+        else:
+            log.error('comps_url must start with git://')
+            return
+        self.cmd(['make'], comps_dir)
+
     def mash(self):
         # TODO: mash in koji
         pass
diff --git a/bootstrap.py b/bootstrap.py
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -93,7 +93,7 @@ def link_system_libs():
     for mod in ('koji', 'rpm', 'OpenSSL', 'urlgrabber', 'pycurl', 'yum',
                 'rpmUtils', 'sqlitecachec', '_sqlitecache', 'psycopg2',
                 'krbVmodule', 'createrepo', 'deltarpm', '_deltarpmmodule',
-                'fedora_cert'):
+                'fedora_cert', 'libxml2', 'libxml2mod'):
         _link_system_lib(mod)
 
 


Test patch:
diff --git a/bodhi/tests/test_masher.py b/bodhi/tests/test_masher.py
--- a/bodhi/tests/test_masher.py
+++ b/bodhi/tests/test_masher.py
@@ -151,6 +151,8 @@ def test_push_invalid_update(self, publish):
         self.masher.consume(msg)
         self.assertEquals(len(publish.call_args_list), 1)
 
+    @mock.patch('bodhi.masher.MasherThread.update_comps')
+    @mock.patch('bodhi.masher.MasherThread.mash')
     @mock.patch('bodhi.masher.MasherThread.sanity_check_repo')
     @mock.patch('bodhi.masher.MasherThread.stage_repo')
     @mock.patch('bodhi.masher.MasherThread.generate_updateinfo')
@@ -175,6 +177,8 @@ def test_update_locking(self, publish, *args):
             up = session.query(Update).one()
             self.assertTrue(up.locked)
 
+    @mock.patch('bodhi.masher.MasherThread.update_comps')
+    @mock.patch('bodhi.masher.MasherThread.mash')
     @mock.patch('bodhi.masher.MasherThread.sanity_check_repo')
     @mock.patch('bodhi.masher.MasherThread.stage_repo')
     @mock.patch('bodhi.masher.MasherThread.generate_updateinfo')
@@ -251,7 +255,13 @@ def test_statefile(self):
         finally:
             t.remove_state()
 
-    def test_testing_digest(self):
+    @mock.patch('bodhi.masher.MasherThread.update_comps')
+    @mock.patch('bodhi.masher.MasherThread.mash')
+    @mock.patch('bodhi.masher.MasherThread.sanity_check_repo')
+    @mock.patch('bodhi.masher.MasherThread.stage_repo')
+    @mock.patch('bodhi.masher.MasherThread.generate_updateinfo')
+    @mock.patch('bodhi.notifications.publish')
+    def test_testing_digest(self, *args):
         t = MasherThread(u'F17', u'testing', [u'bodhi-2.0-1.fc17'],
                          log, self.db_factory, self.tempdir)
         with self.db_factory() as session:
@@ -321,6 +331,8 @@ def test_stage(self):
         link = os.path.join(stage_dir, t.id)
         self.assertTrue(os.path.islink(link))
 
+    @mock.patch('bodhi.masher.MasherThread.update_comps')
+    @mock.patch('bodhi.masher.MasherThread.mash')
     @mock.patch('bodhi.masher.MasherThread.sanity_check_repo')
     @mock.patch('bodhi.masher.MasherThread.stage_repo')
     @mock.patch('bodhi.masher.MasherThread.generate_updateinfo')
@@ -371,6 +383,8 @@ def test_security_update_priority(self, publish, *args):
         self.assertEquals(calls[-1], mock.call(msg={'success': True},
             topic='mashtask.complete'))
 
+    @mock.patch('bodhi.masher.MasherThread.update_comps')
+    @mock.patch('bodhi.masher.MasherThread.mash')
     @mock.patch('bodhi.masher.MasherThread.sanity_check_repo')
     @mock.patch('bodhi.masher.MasherThread.stage_repo')
     @mock.patch('bodhi.masher.MasherThread.generate_updateinfo')
@@ -423,6 +437,8 @@ def test_security_update_priority_testing(self, publish, *args):
         self.assertEquals(calls[-1], mock.call(msg={'success': True},
             topic='mashtask.complete'))
 
+    @mock.patch('bodhi.masher.MasherThread.update_comps')
+    @mock.patch('bodhi.masher.MasherThread.mash')
     @mock.patch('bodhi.masher.MasherThread.sanity_check_repo')
     @mock.patch('bodhi.masher.MasherThread.stage_repo')
     @mock.patch('bodhi.masher.MasherThread.generate_updateinfo')
@@ -471,3 +487,22 @@ def test_security_updates_parallel(self, publish, *args):
             'updates': [u'bodhi-2.0-1.fc18']}, topic='mashtask.mashing'))
         self.assertEquals(calls[2], mock.call(msg={'repo': u'f17-updates',
             'updates': [u'bodhi-2.0-1.fc17']}, topic='mashtask.mashing'))
+
+    @mock.patch('bodhi.masher.MasherThread.mash')
+    @mock.patch('bodhi.masher.MasherThread.sanity_check_repo')
+    @mock.patch('bodhi.masher.MasherThread.stage_repo')
+    @mock.patch('bodhi.masher.MasherThread.generate_updateinfo')
+    @mock.patch('bodhi.masher.MasherThread.cmd')
+    def test_update_comps(self, cmd, *args):
+        self.masher.consume(self.msg)
+        self.assertIn(mock.call(['git', 'pull'], mock.ANY), cmd.mock_calls)
+        self.assertIn(mock.call(['make'], mock.ANY), cmd.mock_calls)
+
+    def test_cmd_failure(self):
+        t = MasherThread(u'F17', u'testing', [u'bodhi-2.0-1.fc17'], log,
+                         self.db_factory, self.tempdir)
+        try:
+            t.cmd('false')
+            assert False
+        except Exception:
+            pass