Repository: fedora-infra/bodhi

Issue:
RepodataException: Required part not in repomd.xml
While testing a compose in staging, the new repo sanity check code was not satisfied with the repository that was generated by Pungi. At this time I am not sure what exactly was unsatisfactory, but this is the traceback I saw:

```
[2018-09-04 21:41:38][fedmsg.consumers   ERROR]f28-updates-testing ComposerThread failed. Transaction rolled back. 
Traceback (most recent call last): 
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 339, in run 
    self.work() 
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 410, in work 
    self._compose_updates() 
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 950, in _compose_updates 
    self._sanity_check_repo() 
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 1154, in _sanity_check_repo 
    sanity_check_repodata(repodata) 
  File "/usr/lib/python2.7/site-packages/bodhi/server/util.py", line 287, in sanity_check_repodata 
    raise RepodataException('Required part not in repomd.xml') 
RepodataException: Required part not in repomd.xml
```


Discussion:
OK, this is strange. Despite this error, if I open a shell and run [```sanity_check_repodata()```](https://github.com/fedora-infra/bodhi/blob/eff7c339baef188deb63035bc90d394659ef866d/bodhi/server/util.py#L284-L342) by hand with the path of the repo that failed above, it passes just fine.

This leads me to suspect that something is not finished writing the files into the repo before ```sanity_check_repodata()``` gets called, but does finish writing it at some point.
I attempted to add a ```gc.collect()``` call before the sanity check runs in the hopes that the garbage collector would clean up the open files, but it did not work around the problem.

It's late in the work day, but my next workaround idea is to find out what open files my process has in the compose directory (```/mnt/koji```) and close them explicitly before running the sanity check.
I met with @puiterwijk this morning to talk about this issue, and he noticed that the issue stems from checking the source repos which do not have all the same files as the RPM repos. For example, source repos do not have ```prestodelta``` data.

The fix will be to adjust the sanity check algorithm so that it doesn't expect some of the data to be present when it is checking a source repo.

Resolving patch:
diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py
--- a/bodhi/server/consumers/masher.py
+++ b/bodhi/server/consumers/masher.py
@@ -1148,10 +1148,11 @@ def _sanity_check_repo(self):
                 if arch == 'source':
                     repodata = os.path.join(self.path, 'compose',
                                             'Everything', arch, 'tree', 'repodata')
+                    sanity_check_repodata(repodata, source=True)
                 else:
                     repodata = os.path.join(self.path, 'compose',
                                             'Everything', arch, 'os', 'repodata')
-                sanity_check_repodata(repodata)
+                    sanity_check_repodata(repodata, source=False)
             except Exception:
                 self.log.exception("Repodata sanity check failed, compose thrown out")
                 self._toss_out_repo()
diff --git a/bodhi/server/util.py b/bodhi/server/util.py
--- a/bodhi/server/util.py
+++ b/bodhi/server/util.py
@@ -245,12 +245,13 @@ def get_critpath_components(collection='master', component_type='rpm', component
     return critpath_components
 
 
-def sanity_check_repodata(myurl):
+def sanity_check_repodata(myurl, source):
     """
     Sanity check the repodata for a given repository.
 
     Args:
         myurl (basestring): A path to a repodata directory.
+        source (bool): True if we are checking a source RPM repository, False otherwise.
     Raises:
         Exception: If the repodata is not valid or does not exist.
     """
@@ -279,7 +280,9 @@ def sanity_check_repodata(myurl):
     hk_repo = hawkey.Repo(myurl)
     try:
         hk_repo.filelists_fn = repo_info['filelists']
-        hk_repo.presto_fn = repo_info['prestodelta']
+        # Source repos don't have DRPMs.
+        if not source:
+            hk_repo.presto_fn = repo_info['prestodelta']
         hk_repo.primary_fn = repo_info['primary']
         hk_repo.repomd_fn = repo_info['repomd']
         hk_repo.updateinfo_fn = repo_info['updateinfo']
@@ -291,14 +294,16 @@ def sanity_check_repodata(myurl):
                            load_presto=True,
                            load_updateinfo=True)
 
-    # Test comps
-    comps = libcomps.Comps()
-    try:
-        ret = comps.fromxml_f(repo_info['group'])
-    except Exception:
-        raise RepodataException('Comps file unable to be parsed')
-    if len(comps.groups) < 1:
-        raise RepodataException('Comps file empty')
+    # Source repos don't have comps.
+    if not source:
+        # Test comps
+        comps = libcomps.Comps()
+        try:
+            ret = comps.fromxml_f(repo_info['group'])
+        except Exception:
+            raise RepodataException('Comps file unable to be parsed')
+        if len(comps.groups) < 1:
+            raise RepodataException('Comps file empty')
 
     # Test updateinfo
     ret = subprocess.call(['zgrep', '<id/>', repo_info['updateinfo']])


Test patch:
diff --git a/bodhi/tests/server/base.py b/bodhi/tests/server/base.py
--- a/bodhi/tests/server/base.py
+++ b/bodhi/tests/server/base.py
@@ -362,7 +362,7 @@ def __call__(self):
             raise
 
 
-def mkmetadatadir(path, updateinfo=None, comps=None):
+def mkmetadatadir(path, updateinfo=None, comps=None, source=False):
     """
     Generate package metadata for a given directory.
 
@@ -374,6 +374,7 @@ def mkmetadatadir(path, updateinfo=None, comps=None):
             No updateinfo is inserted if False is passed. Passing True provides undefined
             behavior.
         comps (basestring or None): The comps to insert instead of example.
+        source (True): If True, do not insert comps or prestodelta. Defaults to False.
     """
     compsfile = '''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE comps PUBLIC "-//Red Hat, Inc.//DTD Comps info//EN" "comps.dtd">
@@ -390,7 +391,7 @@ def mkmetadatadir(path, updateinfo=None, comps=None):
     updateinfofile = ''
     if not os.path.isdir(path):
         os.makedirs(path)
-    if not comps:
+    if not comps and not source:
         comps = os.path.join(path, 'comps.xml')
         with open(comps, 'w') as f:
             f.write(compsfile)
@@ -399,13 +400,13 @@ def mkmetadatadir(path, updateinfo=None, comps=None):
         with open(updateinfo, 'w') as f:
             f.write(updateinfofile)
 
-    subprocess.check_call(['createrepo_c',
-                           '--groupfile', 'comps.xml',
-                           '--deltas',
-                           '--xz',
-                           '--database',
-                           '--quiet',
-                           path])
+    createrepo_command = ['createrepo_c', '--xz', '--database', '--quiet', path]
+
+    if not source:
+        for arg in ('--deltas', 'comps.xml', '--groupfile'):
+            createrepo_command.insert(1, arg)
+
+    subprocess.check_call(createrepo_command)
     if updateinfo is not False:
         metadata.insert_in_repo(createrepo_c.XZ, os.path.join(path, 'repodata'), 'updateinfo',
                                 'xml', os.path.join(path, 'updateinfo.xml'))
diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py
--- a/bodhi/tests/server/consumers/test_masher.py
+++ b/bodhi/tests/server/consumers/test_masher.py
@@ -752,7 +752,8 @@ def test_sanity_check_valid(self, save_state):
             with open(os.path.join(repo, 'Packages', 'a', name), 'w') as tf:
                 tf.write('foo')
 
-        base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree'))
+        base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree'),
+                           source=True)
         os.makedirs(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree', 'Packages',
                                  'a'))
         with open(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree', 'Packages', 'a',
@@ -826,7 +827,8 @@ def test_sanity_check_symlink(self, save_state):
             os.makedirs(os.path.join(repo, 'Packages', 'a'))
             os.symlink('/dev/null', os.path.join(repo, 'Packages', 'a', 'test.notrpm'))
 
-        base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree'))
+        base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree'),
+                           source=True)
         os.makedirs(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree', 'Packages',
                                  'a'))
         os.symlink('/dev/null', os.path.join(t.path, 'compose', 'Everything', 'source', 'tree',
@@ -863,7 +865,8 @@ def test_sanity_check_directories_missing(self, save_state):
             base.mkmetadatadir(repo)
             shutil.rmtree(os.path.join(t.path, 'compose', 'Everything', arch, 'os', 'Packages'))
 
-        base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree'))
+        base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree'),
+                           source=True)
 
         assert 'completed_repo' in t._checkpoints
         save_state.reset_mock()
diff --git a/bodhi/tests/server/test_util.py b/bodhi/tests/server/test_util.py
--- a/bodhi/tests/server/test_util.py
+++ b/bodhi/tests/server/test_util.py
@@ -540,7 +540,7 @@ def test_correct_repo(self):
         base.mkmetadatadir(self.tempdir)
 
         # No exception should be raised here.
-        util.sanity_check_repodata(self.tempdir)
+        util.sanity_check_repodata(self.tempdir, source=False)
 
     def test_updateinfo_empty_tags(self):
         """RepodataException should be raised if <id/> is found in updateinfo."""
@@ -550,7 +550,7 @@ def test_updateinfo_empty_tags(self):
         base.mkmetadatadir(self.tempdir, updateinfo=updateinfo)
 
         with self.assertRaises(util.RepodataException) as exc:
-            util.sanity_check_repodata(self.tempdir)
+            util.sanity_check_repodata(self.tempdir, source=False)
 
         self.assertEqual(str(exc.exception), 'updateinfo.xml.gz contains empty ID tags')
 
@@ -562,7 +562,7 @@ def test_comps_invalid_notxml(self):
         base.mkmetadatadir(self.tempdir, comps=comps)
 
         with self.assertRaises(util.RepodataException) as exc:
-            util.sanity_check_repodata(self.tempdir)
+            util.sanity_check_repodata(self.tempdir, source=False)
 
         self.assertEqual(str(exc.exception), 'Comps file unable to be parsed')
 
@@ -574,7 +574,7 @@ def test_comps_invalid_nonsense(self):
         base.mkmetadatadir(self.tempdir, comps=comps)
 
         with self.assertRaises(util.RepodataException) as exc:
-            util.sanity_check_repodata(self.tempdir)
+            util.sanity_check_repodata(self.tempdir, source=False)
 
         self.assertEqual(str(exc.exception), 'Comps file empty')
 
@@ -592,10 +592,26 @@ def test_repomd_missing_updateinfo(self):
         repomd.write(repomd_path, encoding='UTF-8', xml_declaration=True)
 
         with self.assertRaises(util.RepodataException) as exc:
-            util.sanity_check_repodata(self.tempdir)
+            util.sanity_check_repodata(self.tempdir, source=False)
 
         self.assertEqual(str(exc.exception), 'Required part not in repomd.xml')
 
+    def test_source_true(self):
+        """It should not fail source repos for missing prestodelta or comps."""
+        base.mkmetadatadir(self.tempdir)
+        repomd_path = os.path.join(self.tempdir, 'repodata', 'repomd.xml')
+        repomd = ElementTree.parse(repomd_path)
+        ElementTree.register_namespace('', 'http://linux.duke.edu/metadata/repo')
+        root = repomd.getroot()
+        for data in root.findall('{http://linux.duke.edu/metadata/repo}data'):
+            # Source repos don't have drpms or comps.
+            if data.attrib['type'] in ('group', 'prestodelta'):
+                root.remove(data)
+        repomd.write(repomd_path, encoding='UTF-8', xml_declaration=True)
+
+        # No exception should be raised.
+        util.sanity_check_repodata(self.tempdir, source=True)
+
 
 class TestType2Icon(unittest.TestCase):
     """Test the type2icon() function."""