Repository: pex-tool/pex

Issue:
Top level requirements in PEX-INFO are missing needed environment markers.
The `Requires-Python` wheel METADATA should be converted into the `python_version` environment marker for requirements we store in PEX-INFO to allow for activation of the proper wheel when multiple resolve for different distribution targets. We already correctly resolve all the required distributions:
```
$ python -mpex --python=python2.7 --python=python3.8 zipp -o zipp.pex
DEPRECATION: Python 2.7 reached the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 is no longer maintained. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
$ ./zipp.pex 
Failed to execute PEX file. Needed manylinux2014_x86_64-cp-27-cp27mu compatible dependencies for:
 1: zipp==3.0.0
    But this pex only contains:
      zipp-1.2.0-py2.py3-none-any.whl
      zipp-3.0.0-py3-none-any.whl
$ unzip -qc zipp.pex PEX-INFO | jq .
{
  "always_write_cache": false,
  "build_properties": {
    "class": "CPython",
    "pex_version": "2.1.3",
    "platform": "manylinux2014_x86_64",
    "version": [
      2,
      7,
      17
    ]
  },
  "code_hash": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
  "distributions": {
    "contextlib2-0.6.0.post1-py2.py3-none-any.whl": "3517e14cadfba621f038f09a462a62e56363371a",
    "zipp-1.2.0-py2.py3-none-any.whl": "ac5ea4241acf8a77fc207c4b661cff9ee418ea4e",
    "zipp-3.0.0-py3-none-any.whl": "add980e49f821259d8baae9823d1a0517a1c88b0"
  },
  "emit_warnings": true,
  "ignore_errors": false,
  "inherit_path": "false",
  "interpreter_constraints": [],
  "pex_path": null,
  "requirements": [
    "contextlib2==0.6.0.post1; python_version < \"3.4\"",
    "zipp==1.2.0",
    "zipp==3.0.0"
  ],
  "zip_safe": true
}
```


Discussion:

Resolving patch:
diff --git a/pex/dist_metadata.py b/pex/dist_metadata.py
new file mode 100644
--- /dev/null
+++ b/pex/dist_metadata.py
@@ -0,0 +1,31 @@
+# coding=utf-8
+# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import email
+
+from pex.third_party.packaging.specifiers import SpecifierSet
+from pex.third_party.pkg_resources import DistInfoDistribution
+
+
+def requires_python(dist):
+  """Examines dist for `Python-Requires` metadata and returns version constraints if any.
+
+  See: https://www.python.org/dev/peps/pep-0345/#requires-python
+
+  :param dist: A distribution to check for `Python-Requires` metadata.
+  :type dist: :class:`pkg_resources.Distribution`
+  :return: The required python version specifiers.
+  :rtype: :class:`packaging.specifiers.SpecifierSet` or None
+  """
+  if not dist.has_metadata(DistInfoDistribution.PKG_INFO):
+    return None
+
+  metadata = dist.get_metadata(DistInfoDistribution.PKG_INFO)
+  pkg_info = email.parser.Parser().parsestr(metadata)
+  python_requirement = pkg_info.get('Requires-Python')
+  if not python_requirement:
+    return None
+  return SpecifierSet(python_requirement)
diff --git a/pex/environment.py b/pex/environment.py
--- a/pex/environment.py
+++ b/pex/environment.py
@@ -11,7 +11,7 @@
 import zipfile
 from collections import OrderedDict, defaultdict
 
-from pex import pex_builder, pex_warnings
+from pex import dist_metadata, pex_builder, pex_warnings
 from pex.bootstrap import Bootstrap
 from pex.common import atomic_directory, die, open_zip
 from pex.interpreter import PythonInterpreter
@@ -224,7 +224,14 @@ def can_add(self, dist):
     except ValueError:
       return False
 
-    return not self._supported_tags.isdisjoint(tags.parse_tag(wheel_tags))
+    if self._supported_tags.isdisjoint(tags.parse_tag(wheel_tags)):
+      return False
+
+    python_requires = dist_metadata.requires_python(dist)
+    if not python_requires:
+      return True
+
+    return self._interpreter.identity.version_str in python_requires
 
   def activate(self):
     if not self._activated:
@@ -235,20 +242,28 @@ def activate(self):
     return self._working_set
 
   def _resolve(self, working_set, reqs):
-    reqs_by_key = OrderedDict((req.key, req) for req in reqs)
-    unresolved_reqs = OrderedDict()
-    resolveds = OrderedSet()
-
     environment = self._target_interpreter_env.copy()
     environment['extra'] = list(set(itertools.chain(*(req.extras for req in reqs))))
 
-    # Resolve them one at a time so that we can figure out which ones we need to elide should
-    # there be an interpreter incompatibility.
-    for req in reqs_by_key.values():
+    reqs_by_key = OrderedDict()
+    for req in reqs:
       if req.marker and not req.marker.evaluate(environment=environment):
         TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req)
         continue
-      with TRACER.timed('Resolving %s' % req, V=2):
+      reqs_by_key.setdefault(req.key, []).append(req)
+
+    unresolved_reqs = OrderedDict()
+    resolveds = OrderedSet()
+
+    # Resolve them one at a time so that we can figure out which ones we need to elide should
+    # there be an interpreter incompatibility.
+    for key, reqs in reqs_by_key.items():
+      with TRACER.timed('Resolving {} from {}'.format(key, reqs), V=2):
+        # N.B.: We resolve the bare requirement with no version specifiers since the resolve process
+        # used to build this pex already did so. There may be multiple distributions satisfying any
+        # particular key (e.g.: a Python 2 specific version and a Python 3 specific version for a
+        # multi-python PEX) and we want the working set to pick the most appropriate one.
+        req = Requirement.parse(key)
         try:
           resolveds.update(working_set.resolve([req], env=self))
         except DistributionNotFound as e:
diff --git a/pex/resolver.py b/pex/resolver.py
--- a/pex/resolver.py
+++ b/pex/resolver.py
@@ -11,6 +11,7 @@
 from collections import OrderedDict, defaultdict, namedtuple
 from textwrap import dedent
 
+from pex import dist_metadata
 from pex.common import AtomicDirectory, atomic_directory, safe_mkdtemp
 from pex.distribution_target import DistributionTarget
 from pex.interpreter import spawn_python_job
@@ -20,6 +21,7 @@
 from pex.pip import get_pip
 from pex.platforms import Platform
 from pex.requirements import local_project_from_requirement, local_projects_from_requirement_file
+from pex.third_party.packaging.markers import Marker
 from pex.third_party.pkg_resources import Distribution, Environment, Requirement
 from pex.tracer import TRACER
 from pex.util import CacheHelper
@@ -95,7 +97,27 @@ def __init__(self, markers_by_requirement_key):
 
   def to_requirement(self, dist):
     req = dist.as_requirement()
-    markers = self._markers_by_requirement_key.get(req.key)
+
+    markers = OrderedSet()
+
+    # Here we map any wheel python requirement to the equivalent environment marker:
+    # See:
+    # + https://www.python.org/dev/peps/pep-0345/#requires-python
+    # + https://www.python.org/dev/peps/pep-0508/#environment-markers
+    python_requires = dist_metadata.requires_python(dist)
+    if python_requires:
+      markers.update(
+        Marker(python_version)
+        for python_version in sorted(
+          'python_version {operator} {version!r}'.format(
+            operator=specifier.operator,
+            version=specifier.version
+          ) for specifier in python_requires
+        )
+      )
+
+    markers.update(self._markers_by_requirement_key.get(req.key, ()))
+
     if not markers:
       return req
 
@@ -104,12 +126,12 @@ def to_requirement(self, dist):
       req.marker = marker
       return req
 
-    # Here we have a resolve with multiple paths to the dependency represented by dist. At least
+    # We may have resolved with multiple paths to the dependency represented by dist and at least
     # two of those paths had (different) conditional requirements for dist based on environment
-    # marker predicates. Since the pip resolve succeeded, the implication is that the environment
-    # markers are compatible; i.e.: their intersection selects the target interpreter. Here we
-    # make that intersection explicit.
-    # See: https://www.python.org/dev/peps/pep-0496/#micro-language
+    # marker predicates. In that case, since the pip resolve succeeded, the implication is that the
+    # environment markers are compatible; i.e.: their intersection selects the target interpreter.
+    # Here we make that intersection explicit.
+    # See: https://www.python.org/dev/peps/pep-0508/#grammar
     marker = ' and '.join('({})'.format(marker) for marker in markers)
     return Requirement.parse('{}; {}'.format(req, marker))
 


Test patch:
diff --git a/tests/test_integration.py b/tests/test_integration.py
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -1605,3 +1605,38 @@ def test_trusted_host_handling():
     python=python
   )
   results.assert_success()
+
+
+def test_issues_898():
+  python27 = ensure_python_interpreter(PY27)
+  python36 = ensure_python_interpreter(PY36)
+  with temporary_dir() as td:
+    src_dir = os.path.join(td, 'src')
+    with safe_open(os.path.join(src_dir, 'test_issues_898.py'), 'w') as fp:
+      fp.write(dedent("""
+        import zipp
+
+        print(zipp.__file__)
+      """))
+
+    pex_file = os.path.join(td, 'zipp.pex')
+
+    results = run_pex_command(
+      args=[
+        '--python={}'.format(python27),
+        '--python={}'.format(python36),
+        'zipp>=1,<=3.1.0',
+        '--sources-directory={}'.format(src_dir),
+        '--entry-point=test_issues_898',
+        '-o', pex_file
+      ],
+    )
+    results.assert_success()
+
+    pex_root = os.path.realpath(os.path.join(td, 'pex_root'))
+    for python in python27, python36:
+      output = subprocess.check_output([python, pex_file], env=make_env(PEX_ROOT=pex_root))
+      zipp_location = os.path.realpath(output.decode('utf-8').strip())
+      assert zipp_location.startswith(pex_root), (
+        'Failed to import zipp from {} under {}'.format(pex_file, python)
+      )