Repository: fedora-infra/bodhi

Issue:
Port from yum to dnf
We should port from the `yum` module to `dnf`.

It's currently used in a couple of different places (along with the unit tests):
https://github.com/fedora-infra/bodhi/blob/pyramid/bodhi/util.py#L231
https://github.com/fedora-infra/bodhi/blob/pyramid/bodhi/metadata.py



Discussion:
will do ;)

This is what I have in mind:

``` patch
From 197b2e49abeaf7fb92755c3dc5aa0a6fff87e55e Mon Sep 17 00:00:00 2001
From: Igor Gnatenko <i.gnatenko.brain@gmail.com>
Date: Fri, 30 Jan 2015 18:31:24 +0300
Subject: [PATCH] replace yum with librepo

Signed-off-by: Igor Gnatenko <i.gnatenko.brain@gmail.com>
---
 bodhi/util.py | 50 +++++++++++---------------------------------------
 1 file changed, 11 insertions(+), 39 deletions(-)

diff --git a/bodhi/util.py b/bodhi/util.py
index aa037f0..4bdc1c0 100644
--- a/bodhi/util.py
+++ b/bodhi/util.py
@@ -49,13 +49,6 @@ try:
 except ImportError:
     log.warning("Could not import 'rpm'")

-try:
-    import yum
-    import yum.misc
-except ImportError:
-    log.warning("Could not import 'yum'")
-
-
 _ = TranslationStringFactory('bodhi')

 ## Display a given message as a heading
@@ -231,12 +224,14 @@ class Singleton(object):
 def sanity_check_repodata(myurl):
     """
     Sanity check the repodata for a given repository.
-    Initial implementation by Seth Vidal.
     """

-    import urlgrabber
+    import librepo
+    h = librepo.Handle()
+    r = librepo.Result()
+    h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
+    h.setopt(librepo.LRO_DESTDIR, tempfile.mkdtemp())

-    tempdir = tempfile.mkdtemp()
     errorstrings = []
     if myurl[-1] != '/':
         myurl += '/'
@@ -246,36 +241,13 @@ def sanity_check_repodata(myurl):
     else:
         baseurl = baseurl.replace('repodata/', '/')

-    rf = myurl + 'repomd.xml'
+    h.setopt(librepo.LRO_URLS, [myurl])
+    h.setopt(librepo.LRO_CHECKSUM, True)
     try:
-        rm = urlgrabber.urlopen(rf)
-        repomd = yum.repoMDObject.RepoMD('foo', rm)
-        for t in repomd.fileTypes():
-            data = repomd.getData(t)
-            base, href = data.location
-            if base:
-                loc = base + '/' + href
-            else:
-                loc = baseurl + href
-
-            destfn = tempdir + '/' + os.path.basename(href)
-            dest = urlgrabber.urlgrab(loc, destfn)
-            ctype, known_csum = data.checksum
-            csum = yum.misc.checksum(ctype, dest)
-            if csum != known_csum:
-                errorstrings.append("checksum: %s" % t)
-
-            if href.find('xml') != -1:
-                retcode = subprocess.call(
-                    ['/usr/bin/xmllint', '--noout', dest])
-                if retcode != 0:
-                    errorstrings.append("failed xml read: %s" % t)
-
-    except urlgrabber.grabber.URLGrabError, e:
-        errorstrings.append('Error accessing repository %s' % e)
-
-    if errorstrings:
-        raise RepodataException(','.join(errorstrings))
+        h.perform()
+    except librepo.LibrepoException as e:
+        rc, msg, general_msg = e
+        raise RepodataException(msg)

     updateinfo = os.path.join(myurl, 'updateinfo.xml.gz')
     if os.path.exists(updateinfo):
-- 
2.2.2
```

I not tested it and not sure if it's correctly ;)

btw this issue about switching to librepo instead of dnf I think.

Nice, looks good so far. We've got a unit test for this method, and it's currently exploding when trying to fetch the local repodata.

```
ERROR: test_sanity_check (bodhi.tests.test_masher.TestMasher)                                                   
----------------------------------------------------------------------                                          
Traceback (most recent call last):                                                                              
  File "/home/lmacken/code/github.com/bodhi/bodhi/tests/test_masher.py", line 348, in test_sanity_check         
    t.sanity_check_repo()                                                                                       
  File "/home/lmacken/code/github.com/bodhi/bodhi/masher.py", line 545, in sanity_check_repo                    
    sanity_check_repodata(repodata)                                                                             
  File "/home/lmacken/code/github.com/bodhi/bodhi/util.py", line 252, in sanity_check_repodata                  
    raise RepodataException(msg)                                                                                
RepodataException: Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 
```

I tried prepending `file://` to the path, to no avail.


Resolving patch:
diff --git a/bodhi/util.py b/bodhi/util.py
--- a/bodhi/util.py
+++ b/bodhi/util.py
@@ -21,6 +21,7 @@
 import arrow
 import socket
 import urllib
+import shutil
 import tempfile
 import markdown
 import requests
@@ -74,30 +75,14 @@ def get_nvr(nvr):
     return ['-'.join(x[:-2]), x[-2], x[-1]]
 
 
-def mkmetadatadir(dir):
+def mkmetadatadir(path):
     """
     Generate package metadata for a given directory; if it doesn't exist, then
     create it.
     """
-    log.debug("mkmetadatadir(%s)" % dir)
-    if not isdir(dir):
-        os.makedirs(dir)
-    cache = config.get('createrepo_cache_dir')
-    try:
-        import createrepo
-        conf = createrepo.MetaDataConfig()
-        conf.cachedir = cache
-        conf.outputdir = dir
-        conf.directory = dir
-        conf.quiet = True
-        mdgen = createrepo.MetaDataGenerator(conf)
-        mdgen.doPkgMetadata()
-        mdgen.doRepoMetadata()
-        mdgen.doFinalMove()
-    except ImportError:
-        sys.path.append('/usr/share/createrepo')
-        import genpkgmetadata
-        genpkgmetadata.main(['--cachedir', str(cache), '-q', str(dir)])
+    if not os.path.isdir(path):
+        os.makedirs(path)
+    subprocess.check_call(['createrepo_c',  '--xz', '--database', '--quiet', path])
 
 
 def get_age(date):
diff --git a/bootstrap.py b/bootstrap.py
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -90,10 +90,10 @@ def _link_system_lib(lib):
 
 
 def link_system_libs():
-    for mod in ('koji', 'rpm', 'OpenSSL', 'urlgrabber', 'pycurl', 'yum',
+    for mod in ('koji', 'rpm', 'OpenSSL', 'urlgrabber', 'pycurl',
                 'rpmUtils', 'sqlitecachec', '_sqlitecache', 'psycopg2',
                 'krbVmodule', 'createrepo', 'deltarpm', '_deltarpmmodule',
-                'fedora_cert', 'libxml2', 'libxml2mod', 'librepo'):
+                'fedora_cert', 'libxml2', 'libxml2mod', 'librepo', 'createrepo_c'):
         _link_system_lib(mod)
 
 


Test patch:
diff --git a/bodhi/tests/test_metadata.py b/bodhi/tests/test_metadata.py
--- a/bodhi/tests/test_metadata.py
+++ b/bodhi/tests/test_metadata.py
@@ -22,7 +22,8 @@
 from hashlib import sha256
 from os.path import join, exists, basename
 from sqlalchemy import create_engine
-from yum.update_md import UpdateMetadata
+
+import createrepo_c
 
 from bodhi import log
 from bodhi.config import config
@@ -86,6 +87,11 @@ def _verify_updateinfo(self, repodata):
         assert hash == hashed, "File: %s\nHash: %s" % (basename(updateinfo), hashed)
         return updateinfo
 
+    def get_notice(self, uinfo, title):
+        for record in uinfo.updates:
+            if record.title == title:
+                return record
+
     def test_extended_metadata(self):
         update = self.db.query(Update).one()
 
@@ -103,34 +109,32 @@ def test_extended_metadata(self):
         updateinfo = self._verify_updateinfo(self.repodata)
 
         # Read an verify the updateinfo.xml.gz
-        uinfo = UpdateMetadata()
-        uinfo.add(updateinfo)
-        notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
+        uinfo = createrepo_c.UpdateInfo(updateinfo)
+        notice = self.get_notice(uinfo, 'mutt-1.5.14-1.fc13')
         self.assertIsNone(notice)
 
-        notices = uinfo.get_notices()
-        self.assertEquals(len(notices), 1)
-        notice = notices[0]
+        self.assertEquals(len(uinfo.updates), 1)
+        notice = uinfo.updates[0]
 
         self.assertIsNotNone(notice)
-        self.assertEquals(notice['title'], update.title)
-        self.assertEquals(notice['release'], update.release.long_name)
-        self.assertEquals(notice['status'], update.status.value)
-        self.assertEquals(notice['updated'], update.date_modified)
-        self.assertEquals(notice['from'], str(config.get('bodhi_email')))
-        self.assertEquals(notice['description'], update.notes)
-        self.assertIsNotNone(notice['issued'])
-        self.assertEquals(notice['update_id'], update.alias)
-        self.assertIsNone(notice['epoch'])
-        cve = notice['references'][0]
-        self.assertIsNone(cve['title'])
-        self.assertEquals(cve['type'], 'cve')
-        self.assertEquals(cve['href'], update.cves[0].url)
-        self.assertEquals(cve['id'], update.cves[0].cve_id)
-        bug = notice['references'][1]
-        self.assertEquals(bug['href'], update.bugs[0].url)
-        self.assertEquals(bug['id'], '12345')
-        self.assertEquals(bug['type'], 'bugzilla')
+        self.assertEquals(notice.title, update.title)
+        self.assertEquals(notice.release, update.release.long_name)
+        self.assertEquals(notice.status, update.status.value)
+        self.assertEquals(notice.updated_date, update.date_modified)
+        self.assertEquals(notice.fromstr, str(config.get('bodhi_email')))
+        self.assertEquals(notice.description, update.notes)
+        self.assertIsNotNone(notice.issued_date)
+        self.assertEquals(notice.id, update.alias)
+        #self.assertIsNone(notice.epoch)
+        cve = notice.references[0]
+        self.assertIsNone(cve.title)
+        self.assertEquals(cve.type, 'cve')
+        self.assertEquals(cve.href, update.cves[0].url)
+        self.assertEquals(cve.id, update.cves[0].cve_id)
+        bug = notice.references[1]
+        self.assertEquals(bug.href, update.bugs[0].url)
+        self.assertEquals(bug.id, '12345')
+        self.assertEquals(bug.type, 'bugzilla')
 
     def test_extended_metadata_updating(self):
         update = self.db.query(Update).one()
@@ -151,32 +155,28 @@ def test_extended_metadata_updating(self):
         updateinfo = self._verify_updateinfo(self.repodata)
 
         # Read an verify the updateinfo.xml.gz
-        uinfo = UpdateMetadata()
-        uinfo.add(updateinfo)
-        notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
-        self.assertIsNone(notice)
-
-        notice = uinfo.get_notice(get_nvr(update.title))
+        uinfo = createrepo_c.UpdateInfo(updateinfo)
+        notice = self.get_notice(uinfo, update.title)
 
         self.assertIsNotNone(notice)
-        self.assertEquals(notice['title'], update.title)
-        self.assertEquals(notice['release'], update.release.long_name)
-        self.assertEquals(notice['status'], update.status.value)
-        self.assertEquals(notice['updated'], update.date_modified)
-        self.assertEquals(notice['from'], str(config.get('bodhi_email')))
-        self.assertEquals(notice['description'], update.notes)
-        self.assertIsNotNone(notice['issued'])
-        self.assertEquals(notice['update_id'], update.alias)
-        self.assertIsNone(notice['epoch'])
-        cve = notice['references'][0]
-        self.assertIsNone(cve['title'])
-        self.assertEquals(cve['type'], 'cve')
-        self.assertEquals(cve['href'], update.cves[0].url)
-        self.assertEquals(cve['id'], update.cves[0].cve_id)
-        bug = notice['references'][1]
-        self.assertEquals(bug['href'], update.bugs[0].url)
-        self.assertEquals(bug['id'], '12345')
-        self.assertEquals(bug['type'], 'bugzilla')
+        self.assertEquals(notice.title, update.title)
+        self.assertEquals(notice.release, update.release.long_name)
+        self.assertEquals(notice.status, update.status.value)
+        self.assertEquals(notice.updated_date, update.date_modified)
+        self.assertEquals(notice.fromstr, str(config.get('bodhi_email')))
+        self.assertEquals(notice.description, update.notes)
+        self.assertIsNotNone(notice.issued_date)
+        self.assertEquals(notice.id, update.alias)
+        #self.assertIsNone(notice.epoch)
+        cve = notice.references[0]
+        self.assertIsNone(cve.title)
+        self.assertEquals(cve.type, 'cve')
+        self.assertEquals(cve.href, update.cves[0].url)
+        self.assertEquals(cve.id, update.cves[0].cve_id)
+        bug = notice.references[1]
+        self.assertEquals(bug.href, update.bugs[0].url)
+        self.assertEquals(bug.id, '12345')
+        self.assertEquals(bug.type, 'bugzilla')
 
         # Change the notes on the update, but not the date_modified, so we can
         # ensure that the notice came from the cache
@@ -192,29 +192,11 @@ def test_extended_metadata_updating(self):
         updateinfo = self._verify_updateinfo(self.repodata)
 
         # Read an verify the updateinfo.xml.gz
-        uinfo = UpdateMetadata()
-        uinfo.add(updateinfo)
+        uinfo = createrepo_c.UpdateInfo(updateinfo)
+        notice = self.get_notice(uinfo, update.title)
 
-        notice = uinfo.get_notice(get_nvr(update.title))
         self.assertIsNotNone(notice)
-        self.assertEquals(notice['description'], u'Useful details!')  # not u'x'
-        self.assertEquals(notice['title'], update.title)
-        self.assertEquals(notice['release'], update.release.long_name)
-        self.assertEquals(notice['status'], update.status.value)
-        self.assertEquals(notice['updated'], update.date_modified)
-        self.assertEquals(notice['from'], str(config.get('bodhi_email')))
-        self.assertIsNotNone(notice['issued'])
-        self.assertEquals(notice['update_id'], update.alias)
-        self.assertIsNone(notice['epoch'])
-        cve = notice['references'][0]
-        self.assertIsNone(cve['title'])
-        self.assertEquals(cve['type'], 'cve')
-        self.assertEquals(cve['href'], update.cves[0].url)
-        self.assertEquals(cve['id'], update.cves[0].cve_id)
-        bug = notice['references'][1]
-        self.assertEquals(bug['href'], update.bugs[0].url)
-        self.assertEquals(bug['id'], '12345')
-        self.assertEquals(bug['type'], 'bugzilla')
+        self.assertEquals(notice.description, u'Useful details!')  # not u'x'
 
     def test_metadata_updating_with_edited_update(self):
         update = self.db.query(Update).one()
@@ -235,32 +217,28 @@ def test_metadata_updating_with_edited_update(self):
         updateinfo = self._verify_updateinfo(self.repodata)
 
         # Read an verify the updateinfo.xml.gz
-        uinfo = UpdateMetadata()
-        uinfo.add(updateinfo)
-        notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
-        self.assertIsNone(notice)
-
-        notice = uinfo.get_notice(get_nvr(update.title))
+        uinfo = createrepo_c.UpdateInfo(updateinfo)
+        notice = self.get_notice(uinfo, update.title)
 
         self.assertIsNotNone(notice)
-        self.assertEquals(notice['title'], update.title)
-        self.assertEquals(notice['release'], update.release.long_name)
-        self.assertEquals(notice['status'], update.status.value)
-        self.assertEquals(notice['updated'], update.date_modified)
-        self.assertEquals(notice['from'], str(config.get('bodhi_email')))
-        self.assertEquals(notice['description'], update.notes)
-        self.assertIsNotNone(notice['issued'])
-        self.assertEquals(notice['update_id'], update.alias)
-        self.assertIsNone(notice['epoch'])
-        cve = notice['references'][0]
-        self.assertIsNone(cve['title'])
-        self.assertEquals(cve['type'], 'cve')
-        self.assertEquals(cve['href'], update.cves[0].url)
-        self.assertEquals(cve['id'], update.cves[0].cve_id)
-        bug = notice['references'][1]
-        self.assertEquals(bug['href'], update.bugs[0].url)
-        self.assertEquals(bug['id'], '12345')
-        self.assertEquals(bug['type'], 'bugzilla')
+        self.assertEquals(notice.title, update.title)
+        self.assertEquals(notice.release, update.release.long_name)
+        self.assertEquals(notice.status, update.status.value)
+        self.assertEquals(notice.updated_date, update.date_modified)
+        self.assertEquals(notice.fromstr, str(config.get('bodhi_email')))
+        self.assertEquals(notice.description, update.notes)
+        self.assertIsNotNone(notice.issued_date)
+        self.assertEquals(notice.id, update.alias)
+        #self.assertIsNone(notice.epoch)
+        cve = notice.references[0]
+        self.assertIsNone(cve.title)
+        self.assertEquals(cve.type, 'cve')
+        self.assertEquals(cve.href, update.cves[0].url)
+        self.assertEquals(cve.id, update.cves[0].cve_id)
+        bug = notice.references[1]
+        self.assertEquals(bug.href, update.bugs[0].url)
+        self.assertEquals(bug.id, '12345')
+        self.assertEquals(bug.type, 'bugzilla')
 
         # Change the notes on the update *and* the date_modified
         update.notes = u'x'
@@ -276,13 +254,12 @@ def test_metadata_updating_with_edited_update(self):
         updateinfo = self._verify_updateinfo(self.repodata)
 
         # Read an verify the updateinfo.xml.gz
-        uinfo = UpdateMetadata()
-        uinfo.add(updateinfo)
+        uinfo = createrepo_c.UpdateInfo(updateinfo)
+        notice = self.get_notice(uinfo, update.title)
 
-        notice = uinfo.get_notice(get_nvr(update.title))
         self.assertIsNotNone(notice)
-        self.assertEquals(notice['description'], u'x')
-        self.assertEquals(notice['updated'],
+        self.assertEquals(notice.description, u'x')
+        self.assertEquals(notice.updated_date.strftime('%Y-%m-%d %H:%M:%S'),
                           update.date_modified.strftime('%Y-%m-%d %H:%M:%S'))
 
     def test_metadata_updating_with_old_stable_security(self):
@@ -308,10 +285,8 @@ def test_metadata_updating_with_old_stable_security(self):
         updateinfo = self._verify_updateinfo(self.repodata)
 
         # Read an verify the updateinfo.xml.gz
-        uinfo = UpdateMetadata()
-        uinfo.add(updateinfo)
-
-        notice = uinfo.get_notice(get_nvr(update.title))
+        uinfo = createrepo_c.UpdateInfo(updateinfo)
+        notice = self.get_notice(uinfo, update.title)
         self.assertIsNotNone(notice)
 
         # Create a new non-security update for the same package
@@ -349,13 +324,13 @@ def test_metadata_updating_with_old_stable_security(self):
         updateinfo = self._verify_updateinfo(self.repodata)
 
         # Read an verify the updateinfo.xml.gz
-        uinfo = UpdateMetadata()
-        uinfo.add(updateinfo)
+        uinfo = createrepo_c.UpdateInfo(updateinfo)
 
-        self.assertEquals(len(uinfo.get_notices()), 2)
-        notice = uinfo.get_notice(get_nvr('bodhi-2.0-1.fc17'))
+        self.assertEquals(len(uinfo.updates), 2)
+
+        notice = self.get_notice(uinfo, 'bodhi-2.0-1.fc17')
         self.assertIsNotNone(notice)
-        notice = uinfo.get_notice(get_nvr('bodhi-2.0-2.fc17'))
+        notice = self.get_notice(uinfo, 'bodhi-2.0-2.fc17')
         self.assertIsNotNone(notice)
 
     def test_metadata_updating_with_old_testing_security(self):
@@ -377,10 +352,8 @@ def test_metadata_updating_with_old_testing_security(self):
         updateinfo = self._verify_updateinfo(self.repodata)
 
         # Read an verify the updateinfo.xml.gz
-        uinfo = UpdateMetadata()
-        uinfo.add(updateinfo)
-
-        notice = uinfo.get_notice(get_nvr(update.title))
+        uinfo = createrepo_c.UpdateInfo(updateinfo)
+        notice = self.get_notice(uinfo, update.title)
         self.assertIsNotNone(notice)
 
         # Create a new non-security update for the same package
@@ -421,11 +394,10 @@ def test_metadata_updating_with_old_testing_security(self):
         updateinfo = self._verify_updateinfo(self.repodata)
 
         # Read an verify the updateinfo.xml.gz
-        uinfo = UpdateMetadata()
-        uinfo.add(updateinfo)
+        uinfo = createrepo_c.UpdateInfo(updateinfo)
 
-        self.assertEquals(len(uinfo.get_notices()), 1)
-        notice = uinfo.get_notice(get_nvr('bodhi-2.0-1.fc17'))
+        self.assertEquals(len(uinfo.updates), 1)
+        notice = self.get_notice(uinfo, 'bodhi-2.0-1.fc17')
         self.assertIsNone(notice)
-        notice = uinfo.get_notice(get_nvr('bodhi-2.0-2.fc17'))
+        notice = self.get_notice(uinfo, 'bodhi-2.0-2.fc17')
         self.assertIsNotNone(notice)