Repository: pex-tool/pex

Issue:
Fully pin vendored requirements.
With the introduction of the `packaging` vendored distribution in #831 pex took on its 1st vendored distribution with transitive requirements. Those requirements are not pinned and so re-vendoring over time is not stable. @bradhowes hit this in #852 (https://travis-ci.org/pantsbuild/pex/jobs/640044039#L230) since six released 1.14.0 on January 15th: https://pypi.org/project/six/#history.


Discussion:

Resolving patch:
diff --git a/pex/vendor/__init__.py b/pex/vendor/__init__.py
--- a/pex/vendor/__init__.py
+++ b/pex/vendor/__init__.py
@@ -20,7 +20,8 @@ def _root():
   return path
 
 
-class VendorSpec(collections.namedtuple('VendorSpec', ['key', 'requirement', 'rewrite'])):
+class VendorSpec(collections.namedtuple('VendorSpec',
+                                        ['key', 'requirement', 'rewrite', 'constrain'])):
   """Represents a vendored distribution.
 
   :field str key: The distribution requirement key; e.g.: for a requirement of
@@ -28,6 +29,8 @@ class VendorSpec(collections.namedtuple('VendorSpec', ['key', 'requirement', 're
   :field str requirement: The distribution requirement string; e.g.: requests[security]==2.22.0.
   :field bool rewrite: Whether to re-write the distribution's imports for use with the
     `pex.third_party` importer.
+  :field bool constrain: Whether to attempt to constrain the requirement via pip's --constraint
+    mechanism.
 
   NB: Vendored distributions should comply with the host distribution platform constraints. In the
   case of pex, which is a py2.py3 platform agnostic wheel, vendored libraries should be as well.
@@ -37,7 +40,7 @@ class VendorSpec(collections.namedtuple('VendorSpec', ['key', 'requirement', 're
 
   @classmethod
   def pinned(cls, key, version, rewrite=True):
-    return cls(key=key, requirement='{}=={}'.format(key, version), rewrite=rewrite)
+    return cls(key=key, requirement='{}=={}'.format(key, version), rewrite=rewrite, constrain=True)
 
   @classmethod
   def vcs(cls, url, rewrite=True):
@@ -48,7 +51,8 @@ def vcs(cls, url, rewrite=True):
       raise ValueError('Expected the vcs requirement url to have an #egg=<name> fragment. '
                        'Got: {}'.format(url))
 
-    return cls(key=values[0], requirement=url, rewrite=rewrite)
+    # N.B.: Constraints do not work for vcs urls.
+    return cls(key=values[0], requirement=url, rewrite=rewrite, constrain=False)
 
   @property
   def _subpath_components(self):
diff --git a/pex/vendor/__main__.py b/pex/vendor/__main__.py
--- a/pex/vendor/__main__.py
+++ b/pex/vendor/__main__.py
@@ -191,10 +191,24 @@ def vendorize(root_dir, vendor_specs, prefix):
   for vendor_spec in vendor_specs:
     cmd = ['pip', 'install', '--upgrade', '--no-compile', '--target', vendor_spec.target_dir,
            vendor_spec.requirement]
+
+    constraints_file = os.path.join(vendor_spec.target_dir, 'constraints.txt')
+    if vendor_spec.constrain and os.path.isfile(constraints_file):
+      cmd.extend(['--constraint', constraints_file])
+
     result = subprocess.call(cmd)
     if result != 0:
       raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec))
 
+    if vendor_spec.constrain:
+      cmd = ['pip', 'freeze', '--all', '--path', vendor_spec.target_dir]
+      process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+      stdout, _ = process.communicate()
+      if process.returncode != 0:
+        raise VendorizeError('Failed to freeze vendoring of {!r}'.format(vendor_spec))
+      with open(constraints_file, 'wb') as fp:
+        fp.write(stdout)
+
     # We know we can get these as a by-product of a pip install but never need them.
     safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin'))
     safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py'))
diff --git a/pex/vendor/_vendored/packaging/six.py b/pex/vendor/_vendored/packaging/six.py
--- a/pex/vendor/_vendored/packaging/six.py
+++ b/pex/vendor/_vendored/packaging/six.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2019 Benjamin Peterson
+# Copyright (c) 2010-2020 Benjamin Peterson
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -29,7 +29,7 @@
 import types
 
 __author__ = "Benjamin Peterson <benjamin@python.org>"
-__version__ = "1.13.0"
+__version__ = "1.14.0"
 
 
 # Useful for very coarse version differentiation.
@@ -259,7 +259,7 @@ class _MovedItems(_LazyModule):
     MovedModule("copyreg", "copy_reg"),
     MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
     MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"),
-    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
+    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"),
     MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
     MovedModule("http_cookies", "Cookie", "http.cookies"),
     MovedModule("html_entities", "htmlentitydefs", "html.entities"),
@@ -644,9 +644,11 @@ def u(s):
     if sys.version_info[1] <= 1:
         _assertRaisesRegex = "assertRaisesRegexp"
         _assertRegex = "assertRegexpMatches"
+        _assertNotRegex = "assertNotRegexpMatches"
     else:
         _assertRaisesRegex = "assertRaisesRegex"
         _assertRegex = "assertRegex"
+        _assertNotRegex = "assertNotRegex"
 else:
     def b(s):
         return s
@@ -668,6 +670,7 @@ def indexbytes(buf, i):
     _assertCountEqual = "assertItemsEqual"
     _assertRaisesRegex = "assertRaisesRegexp"
     _assertRegex = "assertRegexpMatches"
+    _assertNotRegex = "assertNotRegexpMatches"
 _add_doc(b, """Byte literal""")
 _add_doc(u, """Text literal""")
 
@@ -684,6 +687,10 @@ def assertRegex(self, *args, **kwargs):
     return getattr(self, _assertRegex)(*args, **kwargs)
 
 
+def assertNotRegex(self, *args, **kwargs):
+    return getattr(self, _assertNotRegex)(*args, **kwargs)
+
+
 if PY3:
     exec_ = getattr(moves.builtins, "exec")
 
@@ -719,16 +726,7 @@ def exec_(_code_, _globs_=None, _locs_=None):
 """)
 
 
-if sys.version_info[:2] == (3, 2):
-    exec_("""def raise_from(value, from_value):
-    try:
-        if from_value is None:
-            raise value
-        raise value from from_value
-    finally:
-        value = None
-""")
-elif sys.version_info[:2] > (3, 2):
+if sys.version_info[:2] > (3,):
     exec_("""def raise_from(value, from_value):
     try:
         raise value from from_value
@@ -808,13 +806,33 @@ def print_(*args, **kwargs):
 _add_doc(reraise, """Reraise an exception.""")
 
 if sys.version_info[0:2] < (3, 4):
+    # This does exactly the same what the :func:`py3:functools.update_wrapper`
+    # function does on Python versions after 3.2. It sets the ``__wrapped__``
+    # attribute on ``wrapper`` object and it doesn't raise an error if any of
+    # the attributes mentioned in ``assigned`` and ``updated`` are missing on
+    # ``wrapped`` object.
+    def _update_wrapper(wrapper, wrapped,
+                        assigned=functools.WRAPPER_ASSIGNMENTS,
+                        updated=functools.WRAPPER_UPDATES):
+        for attr in assigned:
+            try:
+                value = getattr(wrapped, attr)
+            except AttributeError:
+                continue
+            else:
+                setattr(wrapper, attr, value)
+        for attr in updated:
+            getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
+        wrapper.__wrapped__ = wrapped
+        return wrapper
+    _update_wrapper.__doc__ = functools.update_wrapper.__doc__
+
     def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
               updated=functools.WRAPPER_UPDATES):
-        def wrapper(f):
-            f = functools.wraps(wrapped, assigned, updated)(f)
-            f.__wrapped__ = wrapped
-            return f
-        return wrapper
+        return functools.partial(_update_wrapper, wrapped=wrapped,
+                                 assigned=assigned, updated=updated)
+    wraps.__doc__ = functools.wraps.__doc__
+
 else:
     wraps = functools.wraps
 
@@ -919,10 +937,9 @@ def ensure_text(s, encoding='utf-8', errors='strict'):
         raise TypeError("not expecting type '%s'" % type(s))
 
 
-
 def python_2_unicode_compatible(klass):
     """
-    A decorator that defines __unicode__ and __str__ methods under Python 2.
+    A class decorator that defines __unicode__ and __str__ methods under Python 2.
     Under Python 3 it does nothing.
 
     To support Python 2 and 3 with a single code base, define a __str__ method


Test patch: