Repository: enthought/okonomiyaki

Issue:
When repackaging setuptools eggs for Python 3.8 the abi tag is wrong
The abi tag of repackaged setuptools eggs for Python 3.8 is wrong. The enthought metadata of the repackaged egg have abi tag `cp38m` but Python 3.8 has removed the `m` indication.

Discussion:

Patch:
diff --git a/okonomiyaki/file_formats/legacy.py b/okonomiyaki/file_formats/legacy.py
index e88b5f6..bdbc47f 100644
--- a/okonomiyaki/file_formats/legacy.py
+++ b/okonomiyaki/file_formats/legacy.py
@@ -5,7 +5,7 @@ import re
 
 from ..errors import InvalidMetadataField
 from ..platforms import PythonImplementation, default_abi
-
+from ..versions import RuntimeVersion
 
 # To parse the python field in our index and spec/depend
 _PYVER_RE = re.compile(r"(?P<major>\d+)\.(?P<minor>\d+)")
@@ -35,8 +35,11 @@ def _guess_abi_tag(epd_platform, python_tag):
     #
     # In those cases, the mapping (platform pyver) -> ABI is unambiguous,
     # as we only ever used one ABI for a given python version/platform.
-    pyver = _python_tag_to_python(python_tag)
-    return u"cp{0}{1}m".format(pyver[0], pyver[2])
+    pyver = RuntimeVersion.from_string(_python_tag_to_python(python_tag))
+    if pyver >= RuntimeVersion.from_string('3.8'):
+        return u"cp{0}{1}".format(pyver.major, pyver.minor)
+    else:
+        return u"cp{0}{1}m".format(pyver.major, pyver.minor)
 
 
 def _guess_platform_abi(epd_platform, python_tag):
diff --git a/okonomiyaki/file_formats/setuptools_egg.py b/okonomiyaki/file_formats/setuptools_egg.py
index 6d76b19..dc878b0 100644
--- a/okonomiyaki/file_formats/setuptools_egg.py
+++ b/okonomiyaki/file_formats/setuptools_egg.py
@@ -82,7 +82,11 @@ def _guess_abi_from_python(python):
     #
     # In those cases, the mapping (platform pyver) -> ABI is unambiguous,
     # as we only ever used one ABI for a given python version/platform.
-    return "cp{0}{1}m".format(python.major, python.minor)
+    if python.major >= 3 and python.minor >= 8:
+        # Python 3.8 has removed the `m` from the abi tag
+        return u"cp{0}{1}".format(python.major, python.minor)
+    else:
+        return u"cp{0}{1}m".format(python.major, python.minor)
 
 
 def _guess_abi_from_running_python():
@@ -104,7 +108,6 @@ def _guess_abi_from_running_python():
 def _guess_abi(platform, python):
     if platform is None:
         return None
-
     if python is not None:
         if (python.major, python.minor) != sys.version_info[:2]:
             return _guess_abi_from_python(python)
@@ -128,7 +131,6 @@ class SetuptoolsEggMetadata(object):
                  abi_tag=_UNSPECIFIED):
         filename = os.path.basename(path)
         name, version, pyver, platform_string = parse_filename(filename)
-
         if platform is None and platform_string is not None:
             msg = ("Platform-specific egg detected: you need to specify a "
                    "platform argument that is not None to from_egg")


Test patch:
diff --git a/okonomiyaki/file_formats/tests/common.py b/okonomiyaki/file_formats/tests/common.py
index 17b40fe..441c6c3 100644
--- a/okonomiyaki/file_formats/tests/common.py
+++ b/okonomiyaki/file_formats/tests/common.py
@@ -4,8 +4,8 @@ import os.path
 
 from ... import repositories
 
-DATA_DIR = os.path.join(os.path.dirname(repositories.__file__),
-                        "tests", "data")
+DATA_DIR = os.path.join(
+    os.path.dirname(repositories.__file__), "tests", "data")
 
 ENSTALLER_EGG = os.path.join(DATA_DIR, "enstaller-4.5.0-1.egg")
 ETS_EGG = os.path.join(DATA_DIR, "ets-4.3.0-3.egg")
@@ -477,6 +477,13 @@ TRAITS_SETUPTOOLS_EGG = os.path.join(
 
 PIP_SETUPTOOLS_EGG = os.path.join(DATA_DIR, "pip-7.0.3-py3.4.egg")
 
+TRAITS_SETUPTOOLS_OSX_cp38_EGG = os.path.join(
+    DATA_DIR, "traits-6.3.0.dev1702-py3.8-macosx-10.14-x86_64.egg")
+TRAITS_SETUPTOOLS_LINUX_cp38_EGG = os.path.join(
+    DATA_DIR, "traits-6.3.0.dev1702-py3.8-linux-x86_64.egg")
+TRAITS_SETUPTOOLS_WIN_cp38_EGG = os.path.join(
+    DATA_DIR, "traits-6.3.0.dev1702-py3.8-win-amd64.egg")
+
 FAKE_PYSIDE_1_1_0_EGG_PKG_INFO = u"""\
 ============
 About PySide
diff --git a/okonomiyaki/file_formats/tests/test_legacy.py b/okonomiyaki/file_formats/tests/test_legacy.py
index 55bdb02..0202c5d 100644
--- a/okonomiyaki/file_formats/tests/test_legacy.py
+++ b/okonomiyaki/file_formats/tests/test_legacy.py
@@ -10,83 +10,83 @@ from hypothesis.strategies import sampled_from
 
 from okonomiyaki.platforms import EPDPlatform
 
-from ..legacy import _guess_platform_abi
-
-#: Known platforms that at some point have been uploaded into an EDS
-#: server.
-CONFIGURATIONS = {
-    ('cp27', 'osx_x86'): 'darwin',
-    ('cp27', 'osx_x86_64'): 'darwin',
-    ('cp27', 'rh5_x86'): 'gnu',
-    ('cp27', 'rh5_x86_64'): 'gnu',
-    ('cp27', 'win_x86'): 'msvc2008',
-    ('cp27', 'win_x86_64'): 'msvc2008',
-    ('cp30', 'osx_x86'): 'darwin',
-    ('cp30', 'osx_x86_64'): 'darwin',
-    ('cp30', 'rh5_x86'): 'gnu',
-    ('cp30', 'rh5_x86_64'): 'gnu',
-    ('cp30', 'win_x86'): 'msvc2008',
-    ('cp30', 'win_x86_64'): 'msvc2008',
-    ('cp31', 'osx_x86'): 'darwin',
-    ('cp31', 'osx_x86_64'): 'darwin',
-    ('cp31', 'rh5_x86'): 'gnu',
-    ('cp31', 'rh5_x86_64'): 'gnu',
-    ('cp31', 'win_x86'): 'msvc2008',
-    ('cp31', 'win_x86_64'): 'msvc2008',
-    ('cp32', 'osx_x86'): 'darwin',
-    ('cp32', 'osx_x86_64'): 'darwin',
-    ('cp32', 'rh5_x86'): 'gnu',
-    ('cp32', 'rh5_x86_64'): 'gnu',
-    ('cp32', 'win_x86'): 'msvc2008',
-    ('cp32', 'win_x86_64'): 'msvc2008',
-    ('cp33', 'osx_x86'): 'darwin',
-    ('cp33', 'osx_x86_64'): 'darwin',
-    ('cp33', 'rh5_x86'): 'gnu',
-    ('cp33', 'rh5_x86_64'): 'gnu',
-    ('cp33', 'win_x86'): 'msvc2010',
-    ('cp33', 'win_x86_64'): 'msvc2010',
-    ('cp34', 'osx_x86'): 'darwin',
-    ('cp34', 'osx_x86_64'): 'darwin',
-    ('cp34', 'rh5_x86'): 'gnu',
-    ('cp34', 'rh5_x86_64'): 'gnu',
-    ('cp34', 'win_x86'): 'msvc2010',
-    ('cp34', 'win_x86_64'): 'msvc2010',
-    ('cp36', 'osx_x86'): 'darwin',
-    ('cp36', 'osx_x86_64'): 'darwin',
-    ('cp36', 'rh6_x86'): 'gnu',
-    ('cp36', 'rh6_x86_64'): 'gnu',
-    ('cp36', 'rh7_x86'): 'gnu',
-    ('cp36', 'rh7_x86_64'): 'gnu',
-    ('cp36', 'win_x86'): 'msvc2015',
-    ('cp36', 'win_x86_64'): 'msvc2015',
-    ('cp38', 'osx_x86'): 'darwin',
-    ('cp38', 'osx_x86_64'): 'darwin',
-    ('cp38', 'rh7_x86'): 'gnu',
-    ('cp38', 'rh7_x86_64'): 'gnu',
-    ('cp38', 'win_x86'): 'msvc2019',
-    ('cp38', 'win_x86_64'): 'msvc2019',
-    ('ip27', 'osx_x86'): 'darwin',
-    ('ip27', 'osx_x86_64'): 'darwin',
-    ('ip27', 'rh5_x86'): 'gnu',
-    ('ip27', 'rh5_x86_64'): 'gnu',
-    ('ip27', 'win_x86'): 'msvc2008',
-    ('ip27', 'win_x86_64'): 'msvc2008',
-    ('jy27', 'osx_x86'): 'darwin',
-    ('jy27', 'osx_x86_64'): 'darwin',
-    ('jy27', 'rh5_x86'): 'gnu',
-    ('jy27', 'rh5_x86_64'): 'gnu',
-    ('jy27', 'win_x86'): 'msvc2008',
-    ('jy27', 'win_x86_64'): 'msvc2008',
-    ('pp27', 'osx_x86'): 'darwin',
-    ('pp27', 'osx_x86_64'): 'darwin',
-    ('pp27', 'rh5_x86'): 'gnu',
-    ('pp27', 'rh5_x86_64'): 'gnu',
-    ('pp27', 'win_x86'): 'msvc2008',
-    ('pp27', 'win_x86_64'): 'msvc2008'}
+from ..legacy import _guess_platform_abi, _guess_abi_tag
 
 
 class TestGuessPlatformABI(unittest.TestCase):
 
+    #: Known platforms that at some point have been uploaded into an EDS
+    #: server.
+    CONFIGURATIONS = {
+        ('cp27', 'osx_x86'): 'darwin',
+        ('cp27', 'osx_x86_64'): 'darwin',
+        ('cp27', 'rh5_x86'): 'gnu',
+        ('cp27', 'rh5_x86_64'): 'gnu',
+        ('cp27', 'win_x86'): 'msvc2008',
+        ('cp27', 'win_x86_64'): 'msvc2008',
+        ('cp30', 'osx_x86'): 'darwin',
+        ('cp30', 'osx_x86_64'): 'darwin',
+        ('cp30', 'rh5_x86'): 'gnu',
+        ('cp30', 'rh5_x86_64'): 'gnu',
+        ('cp30', 'win_x86'): 'msvc2008',
+        ('cp30', 'win_x86_64'): 'msvc2008',
+        ('cp31', 'osx_x86'): 'darwin',
+        ('cp31', 'osx_x86_64'): 'darwin',
+        ('cp31', 'rh5_x86'): 'gnu',
+        ('cp31', 'rh5_x86_64'): 'gnu',
+        ('cp31', 'win_x86'): 'msvc2008',
+        ('cp31', 'win_x86_64'): 'msvc2008',
+        ('cp32', 'osx_x86'): 'darwin',
+        ('cp32', 'osx_x86_64'): 'darwin',
+        ('cp32', 'rh5_x86'): 'gnu',
+        ('cp32', 'rh5_x86_64'): 'gnu',
+        ('cp32', 'win_x86'): 'msvc2008',
+        ('cp32', 'win_x86_64'): 'msvc2008',
+        ('cp33', 'osx_x86'): 'darwin',
+        ('cp33', 'osx_x86_64'): 'darwin',
+        ('cp33', 'rh5_x86'): 'gnu',
+        ('cp33', 'rh5_x86_64'): 'gnu',
+        ('cp33', 'win_x86'): 'msvc2010',
+        ('cp33', 'win_x86_64'): 'msvc2010',
+        ('cp34', 'osx_x86'): 'darwin',
+        ('cp34', 'osx_x86_64'): 'darwin',
+        ('cp34', 'rh5_x86'): 'gnu',
+        ('cp34', 'rh5_x86_64'): 'gnu',
+        ('cp34', 'win_x86'): 'msvc2010',
+        ('cp34', 'win_x86_64'): 'msvc2010',
+        ('cp36', 'osx_x86'): 'darwin',
+        ('cp36', 'osx_x86_64'): 'darwin',
+        ('cp36', 'rh6_x86'): 'gnu',
+        ('cp36', 'rh6_x86_64'): 'gnu',
+        ('cp36', 'rh7_x86'): 'gnu',
+        ('cp36', 'rh7_x86_64'): 'gnu',
+        ('cp36', 'win_x86'): 'msvc2015',
+        ('cp36', 'win_x86_64'): 'msvc2015',
+        ('cp38', 'osx_x86'): 'darwin',
+        ('cp38', 'osx_x86_64'): 'darwin',
+        ('cp38', 'rh7_x86'): 'gnu',
+        ('cp38', 'rh7_x86_64'): 'gnu',
+        ('cp38', 'win_x86'): 'msvc2019',
+        ('cp38', 'win_x86_64'): 'msvc2019',
+        ('ip27', 'osx_x86'): 'darwin',
+        ('ip27', 'osx_x86_64'): 'darwin',
+        ('ip27', 'rh5_x86'): 'gnu',
+        ('ip27', 'rh5_x86_64'): 'gnu',
+        ('ip27', 'win_x86'): 'msvc2008',
+        ('ip27', 'win_x86_64'): 'msvc2008',
+        ('jy27', 'osx_x86'): 'darwin',
+        ('jy27', 'osx_x86_64'): 'darwin',
+        ('jy27', 'rh5_x86'): 'gnu',
+        ('jy27', 'rh5_x86_64'): 'gnu',
+        ('jy27', 'win_x86'): 'msvc2008',
+        ('jy27', 'win_x86_64'): 'msvc2008',
+        ('pp27', 'osx_x86'): 'darwin',
+        ('pp27', 'osx_x86_64'): 'darwin',
+        ('pp27', 'rh5_x86'): 'gnu',
+        ('pp27', 'rh5_x86_64'): 'gnu',
+        ('pp27', 'win_x86'): 'msvc2008',
+        ('pp27', 'win_x86_64'): 'msvc2008'}
+
     @given(sampled_from(sorted(CONFIGURATIONS)))
     def test_known_cases(self, configuration):
         # given
@@ -96,7 +96,7 @@ class TestGuessPlatformABI(unittest.TestCase):
         result = _guess_platform_abi(epd_platform, configuration[0])
 
         # then
-        expected = CONFIGURATIONS[configuration]
+        expected = self.CONFIGURATIONS[configuration]
         self.assertEqual(
             result, expected,
             msg="{} gives {} instead of {}".format(
@@ -149,3 +149,103 @@ class TestGuessPlatformABI(unittest.TestCase):
 
         # Then
         self.assertEqual(abi, "msvc2008")
+
+
+class TestGuessABITag(unittest.TestCase):
+
+    #: Known platforms that at some point have been uploaded into an EDS
+    #: server. Note that only cpython and pypy implementations are
+    #: supported
+    CONFIGURATIONS = {
+        ('cp27', 'osx_x86'): 'cp27m',
+        ('cp27', 'osx_x86_64'): 'cp27m',
+        ('cp27', 'rh5_x86'): 'cp27m',
+        ('cp27', 'rh5_x86_64'): 'cp27m',
+        ('cp27', 'win_x86'): 'cp27m',
+        ('cp27', 'win_x86_64'): 'cp27m',
+        ('cp30', 'osx_x86'): 'cp30m',
+        ('cp30', 'osx_x86_64'): 'cp30m',
+        ('cp30', 'rh5_x86'): 'cp30m',
+        ('cp30', 'rh5_x86_64'): 'cp30m',
+        ('cp30', 'win_x86'): 'cp30m',
+        ('cp30', 'win_x86_64'): 'cp30m',
+        ('cp31', 'osx_x86'): 'cp31m',
+        ('cp31', 'osx_x86_64'): 'cp31m',
+        ('cp31', 'rh5_x86'): 'cp31m',
+        ('cp31', 'rh5_x86_64'): 'cp31m',
+        ('cp31', 'win_x86'): 'cp31m',
+        ('cp31', 'win_x86_64'): 'cp31m',
+        ('cp32', 'osx_x86'): 'cp32m',
+        ('cp32', 'osx_x86_64'): 'cp32m',
+        ('cp32', 'rh5_x86'): 'cp32m',
+        ('cp32', 'rh5_x86_64'): 'cp32m',
+        ('cp32', 'win_x86'): 'cp32m',
+        ('cp32', 'win_x86_64'): 'cp32m',
+        ('cp33', 'osx_x86'): 'cp33m',
+        ('cp33', 'osx_x86_64'): 'cp33m',
+        ('cp33', 'rh5_x86'): 'cp33m',
+        ('cp33', 'rh5_x86_64'): 'cp33m',
+        ('cp33', 'win_x86'): 'cp33m',
+        ('cp33', 'win_x86_64'): 'cp33m',
+        ('cp34', 'osx_x86'): 'cp34m',
+        ('cp34', 'osx_x86_64'): 'cp34m',
+        ('cp34', 'rh5_x86'): 'cp34m',
+        ('cp34', 'rh5_x86_64'): 'cp34m',
+        ('cp34', 'win_x86'): 'cp34m',
+        ('cp34', 'win_x86_64'): 'cp34m',
+        ('cp36', 'osx_x86'): 'cp36m',
+        ('cp36', 'osx_x86_64'): 'cp36m',
+        ('cp36', 'rh6_x86'): 'cp36m',
+        ('cp36', 'rh6_x86_64'): 'cp36m',
+        ('cp36', 'rh7_x86'): 'cp36m',
+        ('cp36', 'rh7_x86_64'): 'cp36m',
+        ('cp36', 'win_x86'): 'cp36m',
+        ('cp36', 'win_x86_64'): 'cp36m',
+        ('cp38', 'osx_x86'): 'cp38',
+        ('cp38', 'osx_x86_64'): 'cp38',
+        ('cp38', 'rh7_x86'): 'cp38',
+        ('cp38', 'rh7_x86_64'): 'cp38',
+        ('cp38', 'win_x86'): 'cp38',
+        ('cp38', 'win_x86_64'): 'cp38',
+        ('pp27', 'osx_x86'): 'cp27m',
+        ('pp27', 'osx_x86_64'): 'cp27m',
+        ('pp27', 'rh5_x86'): 'cp27m',
+        ('pp27', 'rh5_x86_64'): 'cp27m',
+        ('pp27', 'win_x86'): 'cp27m',
+        ('pp27', 'win_x86_64'): 'cp27m'}
+
+    @given(sampled_from(sorted(CONFIGURATIONS)))
+    def test_known_cases(self, configuration):
+        # given
+        epd_platform = EPDPlatform.from_string(configuration[1])
+
+        # when
+        result = _guess_abi_tag(epd_platform, configuration[0])
+
+        # then
+        expected = self.CONFIGURATIONS[configuration]
+        self.assertEqual(
+            result, expected,
+            msg="{} gives {} instead of {}".format(
+                configuration, result, expected))
+
+    @given(sampled_from(
+        ['cp27', 'cp30', 'cp31', 'cp34', 'cp36', 'cp38']))
+    def test_no_platform(self, python_tag):
+        # When
+        abi = _guess_abi_tag(None, python_tag)
+
+        # Then
+        self.assertIsNone(abi)
+
+    @given(sampled_from(
+        ['rh5-64', 'rh6-64', 'rh7-64', 'win-32', 'win-64', 'osx-64']))
+    def test_no_python_implementation(self, platform_tag):
+        # Given
+        platform = EPDPlatform.from_epd_string(platform_tag)
+
+        # When
+        abi = _guess_abi_tag(platform, None)
+
+        # Then
+        self.assertIsNone(abi)
diff --git a/okonomiyaki/file_formats/tests/test_setuptools_file_format.py b/okonomiyaki/file_formats/tests/test_setuptools_file_format.py
index 8f3708b..1a4ed79 100644
--- a/okonomiyaki/file_formats/tests/test_setuptools_file_format.py
+++ b/okonomiyaki/file_formats/tests/test_setuptools_file_format.py
@@ -5,10 +5,13 @@ if sys.version_info < (2, 7):  # noqa
 else:
     import unittest
 
-from ...errors import OkonomiyakiError
-from ...platforms import EPDPlatform
+from okonomiyaki.errors import OkonomiyakiError
+from okonomiyaki.platforms import EPDPlatform
+from okonomiyaki.versions import RuntimeVersion
 from ..setuptools_egg import SetuptoolsEggMetadata, parse_filename
-from .common import PIP_SETUPTOOLS_EGG, TRAITS_SETUPTOOLS_EGG
+from .common import (
+    PIP_SETUPTOOLS_EGG, TRAITS_SETUPTOOLS_EGG, TRAITS_SETUPTOOLS_OSX_cp38_EGG,
+    TRAITS_SETUPTOOLS_WIN_cp38_EGG, TRAITS_SETUPTOOLS_LINUX_cp38_EGG)
 
 
 class TestParseFilename(unittest.TestCase):
@@ -92,8 +95,7 @@ class TestSetuptoolsEggMetadata(unittest.TestCase):
 
         # When
         metadata = SetuptoolsEggMetadata.from_egg(
-            path, platform, python_tag, abi_tag,
-        )
+            path, platform, python_tag, abi_tag)
 
         # Then
         self.assertEqual(metadata.name, "pip")
@@ -105,9 +107,9 @@ class TestSetuptoolsEggMetadata(unittest.TestCase):
     def test_platform_specific(self):
         # Given
         path = TRAITS_SETUPTOOLS_EGG
+        platform = EPDPlatform.from_epd_string("osx-64")
 
         # When
-        platform = EPDPlatform.from_epd_string("osx-64")
         metadata = SetuptoolsEggMetadata.from_egg(path, platform)
 
         # Then
@@ -120,3 +122,63 @@ class TestSetuptoolsEggMetadata(unittest.TestCase):
         # When/Then
         with self.assertRaises(OkonomiyakiError):
             SetuptoolsEggMetadata.from_egg(path)
+
+    def test_macos_cp38_egg(self):
+        # Given
+        path = TRAITS_SETUPTOOLS_OSX_cp38_EGG
+        python = RuntimeVersion.from_string('3.8.10')
+        platform = EPDPlatform.from_epd_string("osx-64", python)
+
+        # When
+        metadata = SetuptoolsEggMetadata.from_egg(path, platform)
+
+        # Then
+        self.assertEqual(metadata.name, "traits")
+        self.assertEqual(metadata.version, "6.3.0.dev1702")
+        self.assertEqual(metadata.python_tag, "cp38")
+        self.assertEqual(metadata.abi_tag, "cp38")
+        self.assertEqual(metadata.platform_tag, "macosx_10_14_x86_64")
+
+        # When/Then
+        with self.assertRaises(OkonomiyakiError):
+            SetuptoolsEggMetadata.from_egg(path)
+
+    def test_linux_cp38_egg(self):
+        # Given
+        path = TRAITS_SETUPTOOLS_LINUX_cp38_EGG
+        python = RuntimeVersion.from_string('3.8.10')
+        platform = EPDPlatform.from_epd_string("rh7-64", python)
+
+        # When
+        metadata = SetuptoolsEggMetadata.from_egg(path, platform)
+
+        # Then
+        self.assertEqual(metadata.name, "traits")
+        self.assertEqual(metadata.version, "6.3.0.dev1702")
+        self.assertEqual(metadata.python_tag, "cp38")
+        self.assertEqual(metadata.abi_tag, "cp38")
+        self.assertEqual(metadata.platform_tag, "linux_x86_64")
+
+        # When/Then
+        with self.assertRaises(OkonomiyakiError):
+            SetuptoolsEggMetadata.from_egg(path)
+
+    def test_windows_cp38_egg(self):
+        # Given
+        path = TRAITS_SETUPTOOLS_WIN_cp38_EGG
+        python = RuntimeVersion.from_string('3.8.10')
+        platform = EPDPlatform.from_epd_string("win-64", python)
+
+        # When
+        metadata = SetuptoolsEggMetadata.from_egg(path, platform)
+
+        # Then
+        self.assertEqual(metadata.name, "traits")
+        self.assertEqual(metadata.version, "6.3.0.dev1702")
+        self.assertEqual(metadata.python_tag, "cp38")
+        self.assertEqual(metadata.abi_tag, "cp38")
+        self.assertEqual(metadata.platform_tag, "win_amd64")
+
+        # When/Then
+        with self.assertRaises(OkonomiyakiError):
+            SetuptoolsEggMetadata.from_egg(path)