Repository: pex-tool/pex

Issue:
Resolves using `--pex-repository` do not properly canonicalize project names.
Pex current has two resolve paths:
1. Pip
2. PEX files via `--pex-repository`

Pip presumably normalizes project names when resolving requirements as per https://www.python.org/dev/peps/pep-0503/#normalized-names. Pex does not. Right now Pex uses the `pkg_resources` concept of a key via `Requirement.key`:
https://github.com/pantsbuild/pex/blob/8460d6e1ad4802c86ca8bd14407d1a7a486ea522/pex/vendor/_vendored/setuptools/pkg_resources/__init__.py#L3129-L3138
And `Distribution.key`:
https://github.com/pantsbuild/pex/blob/8460d6e1ad4802c86ca8bd14407d1a7a486ea522/pex/vendor/_vendored/setuptools/pkg_resources/__init__.py#L2593-L2601
https://github.com/pantsbuild/pex/blob/8460d6e1ad4802c86ca8bd14407d1a7a486ea522/pex/vendor/_vendored/setuptools/pkg_resources/__init__.py#L2669-L2675
Both of which use `safe_name`:
https://github.com/pantsbuild/pex/blob/8460d6e1ad4802c86ca8bd14407d1a7a486ea522/pex/vendor/_vendored/setuptools/pkg_resources/__init__.py#L1349-L1354

Unfortunately the `pkg_resources` concept of a lower-cased `safe_name` is not quite a PEP-503 normalized name since `.` is not included in the runs of characters converted to `-`.

To provide consistent resolves between Pip and `--pex-repository` Pex should use `packaging`'s `canonicalize_name`:
https://github.com/pantsbuild/pex/blob/196b4cd5b8dd4b4af2586460530e9a777262be7d/pex/vendor/_vendored/packaging/packaging/utils.py#L21-L25




Discussion:

Resolving patch:
diff --git a/pex/environment.py b/pex/environment.py
--- a/pex/environment.py
+++ b/pex/environment.py
@@ -17,6 +17,7 @@
 from pex.distribution_target import DistributionTarget
 from pex.inherit_path import InheritPath
 from pex.orderedset import OrderedSet
+from pex.pep_503 import ProjectName
 from pex.pex_info import PexInfo
 from pex.third_party.packaging import specifiers, tags
 from pex.third_party.pkg_resources import Distribution, Requirement
@@ -74,7 +75,12 @@ def maximum(cls, distribution=None):
 
     def satisfies(self, requirement):
         # type: (Requirement) -> bool
-        return self.distribution in requirement
+        # N.B.: Although Requirement.__contains__ handles Distributions directly, it compares the
+        # Distribution key with the Requirement key and these keys are not properly canonicalized
+        # per PEP-503; so we compare project names here on our own.
+        if ProjectName(self.distribution) != ProjectName(requirement):
+            return False
+        return self.distribution.version in requirement
 
 
 @attr.s(frozen=True)
@@ -154,13 +160,12 @@ class ResolveError(Exception):
 
 
 @attr.s(frozen=True)
-class _RequirementKey(object):
+class _RequirementKey(ProjectName):
     @classmethod
     def create(cls, requirement):
         # type: (Requirement) -> _RequirementKey
-        return cls(requirement.key, frozenset(requirement.extras))
+        return cls(requirement, frozenset(requirement.extras))
 
-    key = attr.ib()  # type: str
     extras = attr.ib()  # type: FrozenSet[str]
 
     def satisfied_keys(self):
@@ -176,7 +181,7 @@ def satisfied_keys(self):
         items = list(self.extras)
         for size in range(len(items) + 1):
             for combination_of_size in itertools.combinations(items, size):
-                yield _RequirementKey(self.key, frozenset(combination_of_size))
+                yield _RequirementKey(self.project_name, frozenset(combination_of_size))
 
 
 class PEXEnvironment(object):
@@ -354,12 +359,12 @@ def __init__(
         self._pex = os.path.realpath(pex)
         self._pex_info = pex_info or PexInfo.from_pex(pex)
 
-        self._available_ranked_dists_by_key = defaultdict(
+        self._available_ranked_dists_by_project_name = defaultdict(
             list
-        )  # type: DefaultDict[str, List[_RankedDistribution]]
-        self._unavailable_dists_by_key = defaultdict(
+        )  # type: DefaultDict[ProjectName, List[_RankedDistribution]]
+        self._unavailable_dists_by_project_name = defaultdict(
             list
-        )  # type: DefaultDict[str, List[_UnrankedDistribution]]
+        )  # type: DefaultDict[ProjectName, List[_UnrankedDistribution]]
         self._resolved_dists = None  # type: Optional[Iterable[Distribution]]
         self._activated_dists = None  # type: Optional[Iterable[Distribution]]
 
@@ -383,11 +388,12 @@ def _update_candidate_distributions(self, distribution_iter):
         # type: (Iterable[Distribution]) -> None
         for dist in distribution_iter:
             ranked_dist = self._can_add(dist)
+            project_name = ProjectName(dist)
             if isinstance(ranked_dist, _RankedDistribution):
                 with TRACER.timed("Adding %s" % dist, V=2):
-                    self._available_ranked_dists_by_key[dist.key].append(ranked_dist)
+                    self._available_ranked_dists_by_project_name[project_name].append(ranked_dist)
             else:
-                self._unavailable_dists_by_key[dist.key].append(ranked_dist)
+                self._unavailable_dists_by_project_name[project_name].append(ranked_dist)
 
     def _can_add(self, dist):
         # type: (Distribution) -> Union[_RankedDistribution, _UnrankedDistribution]
@@ -457,7 +463,9 @@ def _resolve_requirement(
 
         available_distributions = [
             ranked_dist
-            for ranked_dist in self._available_ranked_dists_by_key.get(requirement.key, [])
+            for ranked_dist in self._available_ranked_dists_by_project_name[
+                ProjectName(requirement)
+            ]
             if ranked_dist.satisfies(requirement)
         ]
         if not available_distributions:
@@ -525,35 +533,39 @@ def _root_requirements_iter(self, reqs):
         #   "setuptools==44.1.1; python_version<'3.6'",
         #   "isort==5.6.4; python_version>='3.6'",
         # }
-        qualified_reqs_by_key = OrderedDict()  # type: OrderedDict[str, List[_QualifiedRequirement]]
+        qualified_reqs_by_project_name = (
+            OrderedDict()
+        )  # type: OrderedDict[ProjectName, List[_QualifiedRequirement]]
         for req in reqs:
             required = self._evaluate_marker(req)
             if required is False:
                 continue
-            requirements = qualified_reqs_by_key.get(req.key)
+            project_name = ProjectName(req)
+            requirements = qualified_reqs_by_project_name.get(project_name)
             if requirements is None:
-                qualified_reqs_by_key[req.key] = requirements = []
+                qualified_reqs_by_project_name[project_name] = requirements = []
             requirements.append(_QualifiedRequirement(req, required=required))
 
         # Next, from among the remaining applicable requirements for a given project, we want to
         # select the most tailored (highest ranked) available distribution. That distribution's
         # transitive requirements will later fill in the full resolve.
-        for key, qualified_requirements in qualified_reqs_by_key.items():
-            ranked_dists = self._available_ranked_dists_by_key.get(key)
+        for project_name, qualified_requirements in qualified_reqs_by_project_name.items():
+            ranked_dists = self._available_ranked_dists_by_project_name.get(project_name)
             if ranked_dists is None:
                 # We've winnowed down reqs_by_key to just those requirements whose environment
                 # markers apply; so, we should always have an available distribution.
-                message = "A distribution for {} could not be resolved in this environment.".format(
-                    key
+                message = (
+                    "A distribution for {project_name} could not be resolved in this "
+                    "environment.".format(project_name=project_name)
                 )
-                unavailable_dists = self._unavailable_dists_by_key.get(key)
+                unavailable_dists = self._unavailable_dists_by_project_name.get(project_name)
                 if unavailable_dists:
                     message += (
-                        "Found {count} {distributions} for {key} that do not apply:\n"
+                        "Found {count} {distributions} for {project_name} that do not apply:\n"
                         "{unavailable_dists}".format(
                             count=len(unavailable_dists),
                             distributions=pluralize(unavailable_dists, "distribution"),
-                            key=key,
+                            project_name=project_name,
                             unavailable_dists="\n".join(
                                 "{index}.) {message}".format(
                                     index=index,
@@ -652,7 +664,9 @@ def record_unresolved(dist_not_found):
                         rendered_requirers = "\n    Required by:" "\n      {requirers}".format(
                             requirers="\n      ".join(map(str, requirers))
                         )
-                    contains = self._available_ranked_dists_by_key[requirement.key]
+                    contains = self._available_ranked_dists_by_project_name[
+                        ProjectName(requirement)
+                    ]
                     if contains:
                         rendered_contains = (
                             "\n    But this pex only contains:"
diff --git a/pex/pep_503.py b/pex/pep_503.py
new file mode 100644
--- /dev/null
+++ b/pex/pep_503.py
@@ -0,0 +1,38 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+from pex.third_party.packaging.utils import canonicalize_name
+from pex.third_party.pkg_resources import Distribution, Requirement
+from pex.typing import TYPE_CHECKING, cast
+
+if TYPE_CHECKING:
+    import attr  # vendor:skip
+    from typing import Union
+else:
+    from pex.third_party import attr
+
+
+def _canonicalize_project_name(project_nameable):
+    # type: (Union[Distribution, Requirement, str]) -> str
+    project_name = (
+        project_nameable.project_name
+        if isinstance(project_nameable, (Distribution, Requirement))
+        else project_nameable
+    )
+    return cast(str, canonicalize_name(project_name))
+
+
+@attr.s(frozen=True)
+class ProjectName(object):
+    """Encodes a canonicalized project name as per PEP-502.
+
+    See: https://www.python.org/dev/peps/pep-0503/#normalized-names
+    """
+
+    project_name = attr.ib(converter=_canonicalize_project_name)  # type: str
+
+    def __str__(self):
+        # type: () -> str
+        return self.project_name
diff --git a/pex/resolver.py b/pex/resolver.py
--- a/pex/resolver.py
+++ b/pex/resolver.py
@@ -10,7 +10,6 @@
 import zipfile
 from collections import OrderedDict, defaultdict
 
-from pex import attrs
 from pex.common import AtomicDirectory, atomic_directory, safe_mkdtemp
 from pex.distribution_target import DistributionTarget
 from pex.environment import PEXEnvironment, ResolveError
@@ -18,6 +17,7 @@
 from pex.jobs import Raise, SpawnedJob, execute_parallel
 from pex.network_configuration import NetworkConfiguration
 from pex.orderedset import OrderedSet
+from pex.pep_503 import ProjectName
 from pex.pex_info import PexInfo
 from pex.pip import PackageIndexConfiguration, get_pip
 from pex.platforms import Platform
@@ -653,11 +653,13 @@ def iter_direct_requirements():
                     install_req = build_request.result(built_wheels_dir).finalize_build()
                     yield requirement.as_requirement(dist=install_req.wheel_path)
 
-            direct_requirements_by_key = defaultdict(
+            direct_requirements_by_project_name = defaultdict(
                 OrderedSet
-            )  # type: DefaultDict[str, OrderedSet[Requirement]]
+            )  # type: DefaultDict[ProjectName, OrderedSet[Requirement]]
             for direct_requirement in iter_direct_requirements():
-                direct_requirements_by_key[direct_requirement.key].add(direct_requirement)
+                direct_requirements_by_project_name[ProjectName(direct_requirement)].add(
+                    direct_requirement
+                )
 
         # 3. Install wheels in individual chroots.
 
@@ -706,7 +708,7 @@ def add_installation(install_result):
             distribution = installed_distribution.distribution
             direct_reqs = [
                 req
-                for req in direct_requirements_by_key.get(distribution.key, ())
+                for req in direct_requirements_by_project_name.get(ProjectName(distribution), ())
                 if req
                 and distribution in req
                 and installed_distribution.target.requirement_applies(req)
@@ -734,20 +736,22 @@ def add_installation(install_result):
 
     def _check_install(self, installed_distributions):
         # type: (Iterable[InstalledDistribution]) -> None
-        installed_distribution_by_key = OrderedDict(
-            (resolved_distribution.distribution.key, resolved_distribution)
+        installed_distribution_by_project_name = OrderedDict(
+            (ProjectName(resolved_distribution.distribution), resolved_distribution)
             for resolved_distribution in installed_distributions
-        )
+        )  # type: OrderedDict[ProjectName, ResolvedDistribution]
 
         unsatisfied = []
-        for installed_distribution in installed_distribution_by_key.values():
+        for installed_distribution in installed_distribution_by_project_name.values():
             dist = installed_distribution.distribution
             target = installed_distribution.target
             for requirement in dist.requires():
                 if not target.requirement_applies(requirement):
                     continue
 
-                installed_requirement_dist = installed_distribution_by_key.get(requirement.key)
+                installed_requirement_dist = installed_distribution_by_project_name.get(
+                    ProjectName(requirement)
+                )
                 if not installed_requirement_dist:
                     unsatisfied.append(
                         "{dist} requires {requirement} but no version was resolved".format(
@@ -1354,18 +1358,22 @@ def resolve_from_pex(
     # type: (...) -> List[ResolvedDistribution]
 
     direct_requirements = _parse_reqs(requirements, requirement_files, network_configuration)
-    direct_requirements_by_key = OrderedDict()
+    direct_requirements_by_project_name = (
+        OrderedDict()
+    )  # type: OrderedDict[ProjectName, Requirement]
     for direct_requirement in direct_requirements:
         if isinstance(direct_requirement, LocalProjectRequirement):
             raise Untranslatable(
                 "Cannot resolve local projects from PEX repositories. Asked to resolve {path} "
                 "from {pex}.".format(path=direct_requirement.path, pex=pex)
             )
-        direct_requirements_by_key[
-            direct_requirement.requirement.key
+        direct_requirements_by_project_name[
+            ProjectName(direct_requirement.requirement)
         ] = direct_requirement.requirement
 
-    constraints_by_key = defaultdict(list)  # type: DefaultDict[str, List[Constraint]]
+    constraints_by_project_name = defaultdict(
+        list
+    )  # type: DefaultDict[ProjectName, List[Constraint]]
     if not ignore_errors and (requirement_files or constraint_files):
         fetcher = URLFetcher(network_configuration=network_configuration)
         for location, is_constraints in itertools.chain(
@@ -1376,9 +1384,11 @@ def resolve_from_pex(
                 location, is_constraints=is_constraints, fetcher=fetcher
             ):
                 if isinstance(parsed_item, Constraint):
-                    constraints_by_key[parsed_item.requirement.key].append(parsed_item)
+                    constraints_by_project_name[ProjectName(parsed_item.requirement)].append(
+                        parsed_item
+                    )
 
-    all_reqs = direct_requirements_by_key.values()
+    all_reqs = direct_requirements_by_project_name.values()
     unique_targets = _unique_targets(
         interpreters=interpreters, platforms=platforms, manylinux=manylinux
     )
@@ -1391,13 +1401,14 @@ def resolve_from_pex(
             raise Unsatisfiable(str(e))
 
         for distribution in distributions:
-            direct_requirement = direct_requirements_by_key.get(distribution.key, None)
+            project_name = ProjectName(distribution)
+            direct_requirement = direct_requirements_by_project_name.get(project_name, None)
             if not transitive and not direct_requirement:
                 continue
 
             unmet_constraints = [
                 constraint
-                for constraint in constraints_by_key.get(distribution.key, ())
+                for constraint in constraints_by_project_name.get(project_name, ())
                 if distribution not in constraint.requirement
             ]
             if unmet_constraints:


Test patch:
diff --git a/tests/test_integration.py b/tests/test_integration.py
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -3367,3 +3367,26 @@ def test_invalid_macosx_platform_tag(tmpdir):
     ).assert_success()
 
     subprocess.check_call(args=[setproctitle_pex, "-c", "import setproctitle"])
+
+
+def test_pex_repository_pep503_issues_1302(tmpdir):
+    # type: (Any) -> None
+    repository_pex = os.path.join(str(tmpdir), "repository.pex")
+    with built_wheel(name="foo_bar", version="1.0.0") as wheel_path:
+        run_pex_command(
+            args=[
+                "--no-pypi",
+                "--find-links",
+                os.path.dirname(wheel_path),
+                "Foo._-BAR==1.0.0",
+                "-o",
+                repository_pex,
+            ]
+        ).assert_success()
+
+    foo_bar_pex = os.path.join(str(tmpdir), "foo-bar.pex")
+    run_pex_command(
+        args=["--pex-repository", repository_pex, "Foo._-BAR==1.0.0", "-o", foo_bar_pex]
+    ).assert_success()
+
+    subprocess.check_call(args=[foo_bar_pex, "-c", "import foo_bar"])