Repository: pex-tool/pex

Issue:
Pex probes wheel metadata incorrectly.
I need to find the relevant spec, but the defacto wheel creator appears to lowercase project names when setting up the dist-info metadata directory:
```console
$ python -mvenv PyAthena.venv
$ source PyAthena.venv/bin/activate
(PyAthena.venv) $ pip -q install -U pip
(PyAthena.venv) $ pip wheel --no-deps pyathena==1.11.5
Collecting pyathena==1.11.5
...
Saved ./PyAthena-1.11.5-py2.py3-none-any.whl
(PyAthena.venv) $ zipinfo ./PyAthena-1.11.5-py2.py3-none-any.whl
Archive:  ./PyAthena-1.11.5-py2.py3-none-any.whl
Zip file size: 32074 bytes, number of entries: 19
-rw-r--r--  2.0 unx     1536 b- defN 80-Jan-01 00:00 pyathena/__init__.py
-rw-r--r--  2.0 unx     3343 b- defN 80-Jan-01 00:00 pyathena/async_cursor.py
-rw-r--r--  2.0 unx     2634 b- defN 80-Jan-01 00:00 pyathena/async_pandas_cursor.py
-rw-r--r--  2.0 unx     9619 b- defN 80-Jan-01 00:00 pyathena/common.py
-rw-r--r--  2.0 unx     6391 b- defN 80-Jan-01 00:00 pyathena/connection.py
-rw-r--r--  2.0 unx     4534 b- defN 80-Jan-01 00:00 pyathena/converter.py
-rw-r--r--  2.0 unx     3359 b- defN 80-Jan-01 00:00 pyathena/cursor.py
-rw-r--r--  2.0 unx      718 b- defN 80-Jan-01 00:00 pyathena/error.py
-rw-r--r--  2.0 unx     4493 b- defN 80-Jan-01 00:00 pyathena/formatter.py
-rw-r--r--  2.0 unx     5539 b- defN 80-Jan-01 00:00 pyathena/model.py
-rw-r--r--  2.0 unx     3883 b- defN 80-Jan-01 00:00 pyathena/pandas_cursor.py
-rw-r--r--  2.0 unx    16546 b- defN 80-Jan-01 00:00 pyathena/result_set.py
-rw-r--r--  2.0 unx    14810 b- defN 80-Jan-01 00:00 pyathena/sqlalchemy_athena.py
-rw-r--r--  2.0 unx    10247 b- defN 80-Jan-01 00:00 pyathena/util.py
?rw-r--r--  2.0 unx       79 b- defN 16-Jan-01 00:00 pyathena-1.11.5.dist-info/entry_points.txt
-rw-r--r--  2.0 unx     1055 b- defN 80-Jan-01 00:00 pyathena-1.11.5.dist-info/LICENSE
?rw-r--r--  2.0 unx       87 b- defN 16-Jan-01 00:00 pyathena-1.11.5.dist-info/WHEEL
?rw-r--r--  2.0 unx    32199 b- defN 16-Jan-01 00:00 pyathena-1.11.5.dist-info/METADATA
?rw-r--r--  2.0 unx     1501 b- defN 16-Jan-01 00:00 pyathena-1.11.5.dist-info/RECORD
19 files, 122573 bytes uncompressed, 29662 bytes compressed:  75.8%
```

The upshot is this sort of problem:
```console
$ pex pyathena==1.11.5 -opyathena.pex
Traceback (most recent call last):
  File "/home/jsirois/.venv/pex/bin/pex", line 8, in <module>
    sys.exit(main())
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/bin/pex.py", line 1145, in main
    pex_builder = build_pex(options.requirements, options, cache=ENV.PEX_ROOT)
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/bin/pex.py", line 1023, in build_pex
    resolveds = resolve_multi(
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/resolver.py", line 1054, in resolve_multi
    build_and_install_request.install_distributions(
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/resolver.py", line 697, in install_distributions
    for install_result in execute_parallel(
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/jobs.py", line 431, in execute_parallel
    yield spawn_result.spawned_job.await_result()
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/jobs.py", line 185, in await_result
    job.wait()
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/jobs.py", line 76, in wait
    self._finalize_job()
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/jobs.py", line 110, in _finalize_job
    self._finalizer()
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/pip.py", line 551, in _fixup_install
    project_name_and_version = dist_metadata.project_name_and_version(
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/dist_metadata.py", line 207, in project_name_and_version
    pkg_info = _parse_pkg_info(dist)
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/dist_metadata.py", line 125, in _parse_pkg_info
    pkg_info = _parse_wheel_package_info(dist)
  File "/home/jsirois/.venv/pex/lib/python3.9/site-packages/pex/dist_metadata.py", line 107, in _parse_wheel_package_info
    with whl.open(os.path.join(dist_info_dir, DistInfoDistribution.PKG_INFO)) as fp:
  File "/usr/lib/python3.9/zipfile.py", line 1502, in open
    zinfo = self.getinfo(name)
  File "/usr/lib/python3.9/zipfile.py", line 1429, in getinfo
    raise KeyError(
KeyError: "There is no item named 'PyAthena-1.11.5.dist-info/METADATA' in the archive"
```


Discussion:
Hrm, yeah - something changed in pip / setuptools / wheel land at some point:
```console
^jsirois@gill ~/Downloads $ zipinfo PyAthena-1.9.0-py2.py3-none-any.whl | grep METADATA
-rw-rw-rw-  2.0 fat      672 b- defN 19-Dec-14 11:25 PyAthena-1.9.0.dist-info/METADATA
^jsirois@gill ~/Downloads $ zipinfo PyAthena-1.11.5-py2.py3-none-any.whl | grep METADATA
?rw-r--r--  2.0 unx    32199 b- defN 16-Jan-01 00:00 pyathena-1.11.5.dist-info/METADATA
^jsirois@gill ~/Downloads $
```

Pex will need to do a case-insensitive / fuzzy search for the `.dist-info/` dir.

Resolving patch:
diff --git a/pex/dist_metadata.py b/pex/dist_metadata.py
--- a/pex/dist_metadata.py
+++ b/pex/dist_metadata.py
@@ -5,6 +5,7 @@
 from __future__ import absolute_import
 
 import os
+import re
 import tarfile
 import zipfile
 from contextlib import closing
@@ -22,7 +23,7 @@
 
 if TYPE_CHECKING:
     import attr  # vendor:skip
-    from typing import Dict, Iterator, Optional, Union, List
+    from typing import Dict, Iterable, Iterator, List, Optional, Union
 
     DistributionLike = Union[Distribution, str]
 else:
@@ -97,14 +98,60 @@ def _parse_sdist_package_info(sdist_path):
     return None
 
 
+def find_dist_info_file(
+    project_name,  # type: str
+    version,  # type: str
+    filename,  # type: str
+    listing,  # type: Iterable[str]
+):
+    # type: (...) -> Optional[str]
+
+    # The relevant PEP for project names appears to be the wheel 2.1 spec in PEP-566:
+    #   https://www.python.org/dev/peps/pep-0566/#name
+    #
+    # That defers to PEP-508:
+    #   https://www.python.org/dev/peps/pep-0508/#names
+    #
+    # In practice though it appears the PyPA ecosystem at least does a variant of the name
+    # normalization defined for the simple repository API in PEP-503:
+    #   https://www.python.org/dev/peps/pep-0503/#normalized-names
+    #
+    # For example, with the following setup.py:
+    # ---
+    # from setuptools import setup
+    #
+    # setup(name="Stress-.__Test", version="1.0")
+    #
+    # Using `pip wheel` generates a wheel with a `.dist-info/` dir of `Stress_._Test-1.0.dist-info`.
+    # So `-` -> `_` and runs of `_` go to a single `_`. To be flexible, we accept any run of any
+    # combo of `-`, `_`, and `.` as name component separators.
+    project_name_pattern = re.sub(r"[-_.]+", "[-_.]+", project_name)
+
+    wheel_metadata_pattern = "^{}$".format(
+        os.path.join("{}-{}.dist-info".format(project_name_pattern, re.escape(version)), filename)
+    )
+    wheel_metadata_re = re.compile(wheel_metadata_pattern, re.IGNORECASE)
+    for item in listing:
+        if wheel_metadata_re.match(item):
+            return item
+    return None
+
+
 def _parse_wheel_package_info(wheel_path):
     # type: (str) -> Optional[Message]
     if not wheel_path.endswith(".whl") or not zipfile.is_zipfile(wheel_path):
         return None
     project_name, version, _ = os.path.basename(wheel_path).split("-", 2)
-    dist_info_dir = "{}-{}.dist-info".format(project_name, version)
     with open_zip(wheel_path) as whl:
-        with whl.open(os.path.join(dist_info_dir, DistInfoDistribution.PKG_INFO)) as fp:
+        metadata_file = find_dist_info_file(
+            project_name=project_name,
+            version=version,
+            filename=DistInfoDistribution.PKG_INFO,
+            listing=whl.namelist(),
+        )
+        if not metadata_file:
+            return None
+        with whl.open(metadata_file) as fp:
             return _parse_message(fp.read())
 
 
diff --git a/pex/pip.py b/pex/pip.py
--- a/pex/pip.py
+++ b/pex/pip.py
@@ -2,7 +2,7 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from __future__ import absolute_import, print_function
+from __future__ import absolute_import
 
 import fileinput
 import functools
@@ -29,7 +29,6 @@
 from pex.pex_info import PexInfo
 from pex.platforms import Platform
 from pex.third_party import isolated
-from pex.third_party.pkg_resources import safe_name, safe_version
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
 from pex.util import named_temporary_file
@@ -712,12 +711,6 @@ def _fixup_install(
         project_name_and_version = dist_metadata.project_name_and_version(
             dist
         ) or ProjectNameAndVersion.from_filename(dist)
-        project_name = safe_name(project_name_and_version.project_name).replace("-", "_")
-        version = safe_version(project_name_and_version.version).replace("-", "_")
-        dist_info_dir = os.path.join(
-            install_dir,
-            "{project_name}-{version}.dist-info".format(project_name=project_name, version=version),
-        )
 
         # The `direct_url.json` file is both mandatory for Pip to install and non-hermetic for
         # Pex's purpose, since it contains the absolute local filesystem path to any local wheel
@@ -730,8 +723,21 @@ def _fixup_install(
         # See:
         #   https://www.python.org/dev/peps/pep-0610/
         #   https://packaging.python.org/specifications/direct-url/#specification
-        direct_url_relpath = os.path.join(os.path.basename(dist_info_dir), "direct_url.json")
-        direct_url_abspath = os.path.join(os.path.dirname(dist_info_dir), direct_url_relpath)
+        listing = [
+            os.path.relpath(os.path.join(root, f), install_dir)
+            for root, _, files in os.walk(install_dir)
+            for f in files
+        ]
+        direct_url_relpath = dist_metadata.find_dist_info_file(
+            project_name=project_name_and_version.project_name,
+            version=project_name_and_version.version,
+            filename="direct_url.json",
+            listing=listing,
+        )
+        if not direct_url_relpath:
+            return
+
+        direct_url_abspath = os.path.join(install_dir, direct_url_relpath)
         if not os.path.exists(direct_url_abspath):
             return
 
@@ -743,6 +749,7 @@ def _fixup_install(
 
         # The RECORD is a csv file with the path to each installed file in the 1st column.
         # See: https://www.python.org/dev/peps/pep-0376/#record
+        dist_info_dir = os.path.dirname(direct_url_abspath)
         with closing(
             fileinput.input(files=[os.path.join(dist_info_dir, "RECORD")], inplace=True)
         ) as record_fi:


Test patch:
diff --git a/tests/example_packages/PyAthena-1.11.5-py2.py3-none-any.whl b/tests/example_packages/PyAthena-1.11.5-py2.py3-none-any.whl
new file mode 100644
Binary files /dev/null and b/tests/example_packages/PyAthena-1.11.5-py2.py3-none-any.whl differ
diff --git a/tests/example_packages/PyAthena-1.9.0-py2.py3-none-any.whl b/tests/example_packages/PyAthena-1.9.0-py2.py3-none-any.whl
new file mode 100644
Binary files /dev/null and b/tests/example_packages/PyAthena-1.9.0-py2.py3-none-any.whl differ
diff --git a/tests/test_dist_metadata.py b/tests/test_dist_metadata.py
--- a/tests/test_dist_metadata.py
+++ b/tests/test_dist_metadata.py
@@ -15,11 +15,11 @@
 from pex.dist_metadata import (
     MetadataError,
     ProjectNameAndVersion,
+    find_dist_info_file,
     project_name_and_version,
     requires_dists,
     requires_python,
 )
-from pex.orderedset import OrderedSet
 from pex.pex_warnings import PEXWarning
 from pex.pip import get_pip
 from pex.third_party.packaging.specifiers import SpecifierSet
@@ -275,3 +275,63 @@ def test_requires_dists_none(pygoogleearth_zip_sdist):
             ).format(sdist=sdist)
             == str(warning.message)
         )
+
+
+def test_wheel_metadata_project_name_fuzzy_issues_1375():
+    # type: () -> None
+    with example_distribution("PyAthena-1.9.0-py2.py3-none-any.whl") as (wheel_path, dist):
+        expected = ProjectNameAndVersion("PyAthena", "1.9.0")
+        assert expected == project_name_and_version(wheel_path)
+        assert expected == project_name_and_version(dist)
+
+    with example_distribution("PyAthena-1.11.5-py2.py3-none-any.whl") as (wheel_path, dist):
+        expected = ProjectNameAndVersion("pyathena", "1.11.5")
+        assert expected == project_name_and_version(wheel_path)
+        assert expected == project_name_and_version(dist)
+
+
+def test_find_dist_info_file():
+    # type: () -> None
+    assert (
+        find_dist_info_file(
+            project_name="foo",
+            version="1.0",
+            filename="bar",
+            listing=[],
+        )
+        is None
+    )
+
+    assert (
+        find_dist_info_file(
+            project_name="foo",
+            version="1.0",
+            filename="bar",
+            listing=[
+                "foo-1.0.dist-info/baz",
+            ],
+        )
+        is None
+    )
+
+    assert "Foo-1.0.dist-info/bar" == find_dist_info_file(
+        project_name="foo",
+        version="1.0",
+        filename="bar",
+        listing=[
+            "foo-100.dist-info/bar",
+            "Foo-1.0.dist-info/bar",
+            "foo-1.0.dist-info/bar",
+        ],
+    )
+
+    assert "stress__-.-__Test-1.0rc0.dist-info/direct_url.json" == find_dist_info_file(
+        project_name="Stress-.__Test",
+        version="1.0rc0",
+        filename="direct_url.json",
+        listing=[
+            "direct_url.json",
+            "foo-1.0rc0.dist-info/direct_url.json",
+            "stress__-.-__Test-1.0rc0.dist-info/direct_url.json",
+        ],
+    )