Repository: fedora-infra/bodhi

Issue:
Add a config option to turn off automatic cleaning of old composes
A recent change in Fedora has started to create lots of root owned files in the composes, which causes the [compose to fail](https://pagure.io/fedora-infrastructure/issue/7194) when it tries to clean up old composes. Bodhi recently gained the ability to automatically clean these files, so it is sad, but we must disable the feature in Fedora.

In the interest of semantic versioning, we should add a new setting that allows deployments to disable this feature. The setting should default to cleaning up the old composes.


Discussion:

Resolving patch:
diff --git a/bodhi/server/config.py b/bodhi/server/config.py
--- a/bodhi/server/config.py
+++ b/bodhi/server/config.py
@@ -359,6 +359,9 @@ class BodhiConfig(dict):
         'captcha.ttl': {
             'value': 300,
             'validator': int},
+        'clean_old_composes': {
+            'value': True,
+            'validator': _validate_bool},
         'container.destination_registry': {
             'value': 'registry.fedoraproject.org',
             'validator': six.text_type},
diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py
--- a/bodhi/server/consumers/masher.py
+++ b/bodhi/server/consumers/masher.py
@@ -431,9 +431,10 @@ def work(self):
             self.check_all_karma_thresholds()
             self.obsolete_older_updates()
 
-            # Clean old composes
-            self.save_state(ComposeState.cleaning)
-            clean_old_mashes.remove_old_composes()
+            if config['clean_old_composes']:
+                # Clean old composes
+                self.save_state(ComposeState.cleaning)
+                clean_old_mashes.remove_old_composes()
 
             self.save_state(ComposeState.success)
             self.success = True


Test patch:
diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py
--- a/bodhi/tests/server/consumers/test_masher.py
+++ b/bodhi/tests/server/consumers/test_masher.py
@@ -1206,6 +1206,7 @@ def test_mash_late_exit(self, publish, *args):
             self.assertEqual(compose.error_message, 'Pungi exited with status 1')
         self.assertEqual(t._checkpoints, {'determine_and_perform_tag_actions': True})
 
+    @mock.patch.dict('bodhi.server.consumers.masher.config', {'clean_old_composes': False})
     @mock.patch(**mock_taskotron_results)
     @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo')
     @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo')
@@ -1213,7 +1214,102 @@ def test_mash_late_exit(self, publish, *args):
     @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync')
     @mock.patch('bodhi.server.scripts.clean_old_mashes.NUM_TO_KEEP', 2)
     @mock.patch('bodhi.server.notifications.publish')
-    def test_mash(self, publish, *args):
+    def test_clean_old_composes_false(self, publish, *args):
+        """Test work() with clean_old_composes set to False."""
+        self.expected_sems = 1
+
+        # Set the request to stable right out the gate so we can test gating
+        self.set_stable_request(u'bodhi-2.0-1.fc17')
+        msg = self._make_msg()
+        mash_dir = os.path.join(self.tempdir, 'cool_dir')
+
+        # Set up some directories that look similar to what might be found in production, with
+        # some directories that don't match the pattern of ending in -<timestamp>.
+        dirs = {
+            'dist-5E-epel-161003.0724', 'dist-5E-epel-161011.0458', 'dist-5E-epel-161012.1854',
+            'dist-5E-epel-161013.1711', 'dist-5E-epel-testing-161001.0424',
+            'dist-5E-epel-testing-161003.0856', 'dist-5E-epel-testing-161006.0053',
+            'dist-6E-epel-161002.2331', 'dist-6E-epel-161003.2046',
+            'dist-6E-epel-testing-161001.0528', 'epel7-161003.0724', 'epel7-161003.2046',
+            'epel7-161004.1423', 'epel7-161005.1122', 'epel7-testing-161001.0424',
+            'epel7-testing-161003.0621', 'epel7-testing-161003.2217', 'f23-updates-161002.2331',
+            'f23-updates-161003.1302', 'f23-updates-161004.1423', 'f23-updates-161005.0259',
+            'f23-updates-testing-161001.0424', 'f23-updates-testing-161003.0621',
+            'f23-updates-testing-161003.2217', 'f24-updates-161002.2331',
+            'f24-updates-161003.1302', 'f24-updates-testing-161001.0424',
+            'this_should_get_left_alone', 'f23-updates-should_be_untouched',
+            'f23-updates.repocache', 'f23-updates-testing-blank'}
+        [os.makedirs(os.path.join(mash_dir, d)) for d in dirs]
+        # Now let's make a few files here and there.
+        with open(os.path.join(mash_dir, 'dist-5E-epel-161003.0724', 'oops.txt'), 'w') as oops:
+            oops.write('This mash failed to get cleaned and left this file around, oops!')
+        with open(os.path.join(mash_dir, 'COOL_FILE.txt'), 'w') as cool_file:
+            cool_file.write('This file should be allowed to hang out here because it\'s cool.')
+
+        t = RPMComposerThread(self.semmock, msg['body']['msg']['composes'][0],
+                              'ralph', log, self.db_factory, mash_dir)
+
+        with self.db_factory() as session:
+            with mock.patch('bodhi.server.consumers.masher.subprocess.Popen') as Popen:
+                with mock.patch.dict(config, {'mash_dir': mash_dir}):
+                    release = session.query(Release).filter_by(name=u'F17').one()
+                    Popen.side_effect = self._generate_fake_pungi(t, 'stable_tag', release)
+                    t.run()
+
+        actual_dirs = set([
+            d for d in os.listdir(mash_dir)
+            if os.path.isdir(os.path.join(mash_dir, d)) and not d.startswith("Fedora-17-updates")])
+
+        # No dirs should have been removed since we had clean_old_composes set False.
+        self.assertEqual(actual_dirs, dirs)
+        # The cool file should still be here
+        actual_files = [f for f in os.listdir(mash_dir)
+                        if os.path.isfile(os.path.join(mash_dir, f))]
+        self.assertEqual(actual_files, ['COOL_FILE.txt'])
+
+        # Also, ensure we reported success
+        publish.assert_called_with(topic="mashtask.complete",
+                                   force=True,
+                                   msg=dict(success=True,
+                                            repo='f17-updates',
+                                            ctype='rpm',
+                                            agent='ralph'))
+        publish.assert_any_call(topic='update.complete.stable',
+                                force=True,
+                                msg=mock.ANY)
+
+        self.assertEqual(
+            Popen.mock_calls,
+            [mock.call(
+                [config['pungi.cmd'], '--config', '{}/pungi.conf'.format(t._pungi_conf_dir),
+                 '--quiet', '--print-output-dir', '--target-dir', t.mash_dir, '--old-composes',
+                 t.mash_dir, '--no-latest-link', '--label', t._label],
+                cwd=t.mash_dir, shell=False, stderr=-1,
+                stdin=mock.ANY,
+                stdout=mock.ANY)])
+        d = datetime.datetime.utcnow()
+        self.assertEqual(
+            t._checkpoints,
+            {'completed_repo': os.path.join(
+                mash_dir, 'Fedora-17-updates-{}{:02}{:02}.0'.format(d.year, d.month, d.day)),
+             'compose_done': True,
+             'determine_and_perform_tag_actions': True,
+             'modify_bugs': True,
+             'send_stable_announcements': True,
+             'send_testing_digest': True,
+             'status_comments': True})
+        self.assertTrue(os.path.exists(mash_dir))
+
+    @mock.patch.dict('bodhi.server.consumers.masher.config', {'clean_old_composes': True})
+    @mock.patch(**mock_taskotron_results)
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_repo_signature')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync')
+    @mock.patch('bodhi.server.scripts.clean_old_mashes.NUM_TO_KEEP', 2)
+    @mock.patch('bodhi.server.notifications.publish')
+    def test_clean_old_composes_true(self, publish, *args):
+        """Test work() with clean_old_composes set to True."""
         self.expected_sems = 1
 
         # Set the request to stable right out the gate so we can test gating
@@ -1310,6 +1406,64 @@ def test_mash(self, publish, *args):
              'status_comments': True})
         self.assertTrue(os.path.exists(mash_dir))
 
+    @mock.patch(**mock_taskotron_results)
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_repo_signature')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync')
+    @mock.patch('bodhi.server.scripts.clean_old_mashes.NUM_TO_KEEP', 2)
+    @mock.patch('bodhi.server.notifications.publish')
+    def test_mash(self, publish, *args):
+        self.expected_sems = 1
+
+        # Set the request to stable right out the gate so we can test gating
+        self.set_stable_request(u'bodhi-2.0-1.fc17')
+        msg = self._make_msg()
+        mash_dir = os.path.join(self.tempdir, 'cool_dir')
+
+        t = RPMComposerThread(self.semmock, msg['body']['msg']['composes'][0],
+                              'ralph', log, self.db_factory, mash_dir)
+
+        with self.db_factory() as session:
+            with mock.patch('bodhi.server.consumers.masher.subprocess.Popen') as Popen:
+                with mock.patch.dict(config, {'mash_dir': mash_dir}):
+                    release = session.query(Release).filter_by(name=u'F17').one()
+                    Popen.side_effect = self._generate_fake_pungi(t, 'stable_tag', release)
+                    t.run()
+
+        # Ensure we reported success
+        publish.assert_called_with(topic="mashtask.complete",
+                                   force=True,
+                                   msg=dict(success=True,
+                                            repo='f17-updates',
+                                            ctype='rpm',
+                                            agent='ralph'))
+        publish.assert_any_call(topic='update.complete.stable',
+                                force=True,
+                                msg=mock.ANY)
+
+        self.assertEqual(
+            Popen.mock_calls,
+            [mock.call(
+                [config['pungi.cmd'], '--config', '{}/pungi.conf'.format(t._pungi_conf_dir),
+                 '--quiet', '--print-output-dir', '--target-dir', t.mash_dir, '--old-composes',
+                 t.mash_dir, '--no-latest-link', '--label', t._label],
+                cwd=t.mash_dir, shell=False, stderr=-1,
+                stdin=mock.ANY,
+                stdout=mock.ANY)])
+        d = datetime.datetime.utcnow()
+        self.assertEqual(
+            t._checkpoints,
+            {'completed_repo': os.path.join(
+                mash_dir, 'Fedora-17-updates-{}{:02}{:02}.0'.format(d.year, d.month, d.day)),
+             'compose_done': True,
+             'determine_and_perform_tag_actions': True,
+             'modify_bugs': True,
+             'send_stable_announcements': True,
+             'send_testing_digest': True,
+             'status_comments': True})
+        self.assertTrue(os.path.exists(mash_dir))
+
     @mock.patch(**mock_taskotron_results)
     @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo')
     @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo')