Repository: pex-tool/pex

Issue:
Improve discoverability in the presence of conflicting requirements
In PEX 0.8.6, it's easy to get into a situation like this where two requirements have transitive dependencies that conflict. Here is an example, generated from pants inside of Twitter:

```
...
16:44:59 00:11     [pytest]
16:44:59 00:11       [run]
                     Building chroot for [PythonTests(BuildFileAddress(/Users/areitz/workspace/source/science/tests/python/twitter/source_eng/snapshot/BUILD, snapshot))]:
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/contextutil/BUILD, contextutil))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/dirutil/BUILD, dirutil))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/git/BUILD, git))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/source_eng/snapshot/BUILD, lib))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/app/BUILD, app))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/collections/BUILD, collections))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/lang/BUILD, lang))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/log/BUILD, log))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/options/BUILD, options))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/process/BUILD, process))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/util/BUILD, util))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/app/modules/BUILD, modules))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/fs/BUILD, fs))
  Dumping library: PythonLibrary(BuildFileAddress(/Users/areitz/workspace/source/science/src/python/twitter/common/quantity/BUILD, quantity))
  Dumping requirement: twitter.common.contextutil==0.3.2
  Dumping requirement: twitter.common.dirutil==0.3.2
  Dumping requirement: twitter.common.collections==0.3.2
  Dumping requirement: twitter.common.lang==0.3.2
  Dumping requirement: twitter.common.log==0.3.2
  Dumping requirement: twitter.common.options==0.3.2
  Dumping requirement: twitter.common.process==0.3.2
  Dumping requirement: twitter.common.util==0.3.2
  Dumping requirement: twitter.common.fs==0.3.2
  Dumping requirement: twitter.common.quantity==0.3.2
  Dumping requirement: arrow==0.5.4
  Dumping requirement: six==1.4.1
  Dumping requirement: mock==1.0.1
  Dumping requirement: pytest-timeout==0.3
  Dumping requirement: setuptools==1.1.7
  Dumping requirement: pytest
  Dumping requirement: pytest-timeout
  Dumping requirement: pytest-cov
  Dumping requirement: unittest2
Skipping PythonRequirement(unittest2py3k) based upon version filter

               FAILURE

Exception caught:
  File "/Users/areitz/.pants.d/bin/pants.pex/pants-1.1.0-testing-py27.pex/.bootstrap/_pex/pex.py", line 271, in execute
    self.execute_entry(entry_point, args)
  File "/Users/areitz/.pants.d/bin/pants.pex/pants-1.1.0-testing-py27.pex/.bootstrap/_pex/pex.py", line 320, in execute_entry
    runner(entry_point)
  File "/Users/areitz/.pants.d/bin/pants.pex/pants-1.1.0-testing-py27.pex/.bootstrap/_pex/pex.py", line 343, in execute_pkg_resources
    runner()
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/bin/pants_exe.py", line 81, in main
    _run(exiter)
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/bin/pants_exe.py", line 74, in _run
    result = goal_runner.run()
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/bin/goal_runner.py", line 174, in run
    result = self._do_run()
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/bin/goal_runner.py", line 230, in _do_run
    return engine.execute(context, self.goals)
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/engine/engine.py", line 26, in execute
    self.attempt(context, goals)
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/engine/round_engine.py", line 212, in attempt
    goal_executor.attempt(explain)
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/engine/round_engine.py", line 45, in attempt
    task.execute()
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/backend/python/tasks/pytest_run.py", line 108, in execute
    if self.run_tests(test_targets, workunit):
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/backend/python/tasks/pytest_run.py", line 113, in run_tests
    return 0 if self._do_run_tests(targets, workunit).success else 1
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/backend/python/tasks/pytest_run.py", line 356, in _do_run_tests
    with self._test_runner(targets, workunit) as (pex, test_args):
  File "/opt/twitter/Cellar/python/2.7.9/Frameworks/Python.framework/Versions/2.7/lib/python2.7/contextlib.py", line 17, in __enter__
    return self.gen.next()
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/backend/python/tasks/pytest_run.py", line 336, in _test_runner
    builder = chroot.dump()
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/backend/python/python_chroot.py", line 216, in dump
    find_links=find_links)
  File "/Users/areitz/workspace/source/science/.pex/install/pantsbuild.pants-0.0.32-py2-none-any.whl.69fb72b6af1d39fa7822b427eed13767627a68b9/pantsbuild.pants-0.0.32-py2-none-any.whl/pants/backend/python/resolver.py", line 68, in resolve_multi
    cache_ttl=ttl)
  File "/Users/areitz/workspace/source/science/.pex/install/pex-0.8.6-py2.py3-none-any.whl.3cbdb5d909a2f0a8f8d3e8a8dc123e44a80b45a6/pex-0.8.6-py2.py3-none-any.whl/pex/resolver.py", line 220, in resolve
    raise Unsatisfiable('Cannot satisfy requirements: %s' % requirement_set[requirement.key])

Exception message: Cannot satisfy requirements: [PythonRequirement(six==1.4.1), Requirement.parse('six>=1.5')]
```

In this case, the target has a direct dependency on six 1.4.1, but some other dependency is bringing in a conflicting requirement transitively. It's really hard to find this, even with debugging turned on, because PEX doesn't print out what additional requirements it's adding to the set, and where it's getting them from. I added this small patch:

```
diff --git a/.deps/pex-0.8.6-py2.py3-none-any.whl/pex/resolver.py b/.deps/pex-0.8.6-py2.py3-none-any.whl/pex/resolver.py
index 98aa7b8..4ca92c4 100644
--- a/.deps/pex-0.8.6-py2.py3-none-any.whl/pex/resolver.py
+++ b/.deps/pex-0.8.6-py2.py3-none-any.whl/pex/resolver.py
@@ -228,6 +228,8 @@ def resolve(
           continue
         new_requirements.update(requires(highest_package, requirement))
         processed_requirements.add(requirement)
+      if new_requirements:
+        TRACER.log('Requirement "{}" brought in extra requirements: {}'.format(requirement_key, new_requirements))
       requirements.extend(list(new_requirements))

     if not requirements:
```

Which adds the following output:

```
pex: Requirement "twitter.common.contextutil" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.dirutil==0.3.2')])
pex: Requirement "pytest-timeout" brought in extra requirements: OrderedSet([Requirement.parse('pytest>=2.0')])
pex: Requirement "twitter.common.collections" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.lang==0.3.2')])
pex: Requirement "twitter.common.quantity" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.lang==0.3.2')])
pex: Requirement "twitter.common.process" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.string==0.3.2')])
pex: Requirement "twitter.common.util" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.contextutil==0.3.2')])
pex: Requirement "arrow" brought in extra requirements: OrderedSet([Requirement.parse('python-dateutil')])
pex: Requirement "pytest" brought in extra requirements: OrderedSet([Requirement.parse('py>=1.4.25')])
pex: Requirement "twitter.common.log" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.dirutil==0.3.2'), Requirement.parse('twitter.common.options==0.3.2')])
pex: Requirement "twitter.common.dirutil" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.lang==0.3.2')])
pex: Requirement "twitter.common.fs" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.util==0.3.2'), Requirement.parse('twitter.common.quantity==0.3.2'), Requirement.parse('twitter.common.string==0.3.2')])
pex: Requirement "pytest-cov" brought in extra requirements: OrderedSet([Requirement.parse('pytest>=2.2.3'), Requirement.parse('cov-core>=1.6')])
pex: Requirement "twitter.common.contextutil" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.dirutil==0.3.2')])
pex: Requirement "pytest" brought in extra requirements: OrderedSet([Requirement.parse('py>=1.4.25')])
pex: Requirement "twitter.common.string" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.lang==0.3.2')])
pex: Requirement "twitter.common.util" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.contextutil==0.3.2')])
pex: Requirement "python-dateutil" brought in extra requirements: OrderedSet([Requirement.parse('six>=1.5')])
pex: Requirement "twitter.common.quantity" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.lang==0.3.2')])
pex: Requirement "cov-core" brought in extra requirements: OrderedSet([Requirement.parse('coverage>=3.4')])
pex: Requirement "twitter.common.dirutil" brought in extra requirements: OrderedSet([Requirement.parse('twitter.common.lang==0.3.2')])
```

This makes it trivially easy to see that the floating dep on `python-dateutil` is causing the conflict.

I went to submit this patch upstream, but this code has totally changed. I'll wait until we're consuming PEX 1.0 inside Twitter, and once I have something that runs in our enviornment, I'll see if this patch can be re-applied.



Discussion:

Resolving patch:
diff --git a/pex/bin/pex.py b/pex/bin/pex.py
--- a/pex/bin/pex.py
+++ b/pex/bin/pex.py
@@ -30,7 +30,7 @@
 from pex.platforms import Platform
 from pex.requirements import requirements_from_file
 from pex.resolvable import Resolvable
-from pex.resolver import CachingResolver, Resolver
+from pex.resolver import CachingResolver, Resolver, Unsatisfiable
 from pex.resolver_options import ResolverOptionsBuilder
 from pex.tracer import TRACER, TraceLogger
 from pex.variables import Variables
@@ -467,7 +467,10 @@ def build_pex(args, options, resolver_option_builder):
     resolver = Resolver(**resolver_kwargs)
 
   with TRACER.timed('Resolving distributions'):
-    resolveds = resolver.resolve(resolvables)
+    try:
+      resolveds = resolver.resolve(resolvables)
+    except Unsatisfiable as e:
+      die(e)
 
   for dist in resolveds:
     log('  %s' % dist, v=options.verbosity)
diff --git a/pex/resolver.py b/pex/resolver.py
--- a/pex/resolver.py
+++ b/pex/resolver.py
@@ -6,6 +6,7 @@
 import os
 import shutil
 import time
+from collections import namedtuple
 
 from .common import safe_mkdir
 from .fetcher import Fetcher
@@ -39,12 +40,23 @@ def iter(self, req):
         yield package
 
 
-class _ResolvableSet(object):
-  class Error(Exception): pass
-  class Unsatisfiable(Error): pass
+class _ResolvedPackages(namedtuple('_ResolvedPackages', 'resolvable packages parent')):
+  @classmethod
+  def empty(cls):
+    return cls(None, OrderedSet(), None)
+
+  def merge(self, other):
+    if other.resolvable is None:
+      return _ResolvedPackages(self.resolvable, self.packages, self.parent)
+    return _ResolvedPackages(
+        self.resolvable,
+        self.packages & other.packages,
+        self.parent)
 
+
+class _ResolvableSet(object):
   def __init__(self):
-    # Pairs of (resolvable, resolved_packages)
+    # A list of _ResolvedPackages
     self.__tuples = []
 
   def _collapse(self):
@@ -57,28 +69,39 @@ def _collapse(self):
     # resolvable wins" but it seems highly unlikely this will materially affect anybody
     # adversely but could be the source of subtle resolution quirks.
     resolvables = {}
-    for resolvable, packages in self.__tuples:
-      first_resolvable, old_packages = resolvables.get(resolvable.name, (None, OrderedSet()))
-      if not first_resolvable:
-        resolvables[resolvable.name] = (resolvable, OrderedSet(packages))
+    for resolved_packages in self.__tuples:
+      key = resolved_packages.resolvable.name
+      previous = resolvables.get(key, _ResolvedPackages.empty())
+      if previous.resolvable is None:
+        resolvables[key] = resolved_packages
       else:
-        resolvables[resolvable.name] = (first_resolvable, old_packages & packages)
+        resolvables[key] = previous.merge(resolved_packages)
     return resolvables
 
+  def _synthesize_parents(self, name):
+    def render_resolvable(resolved_packages):
+      return '%s%s' % (
+          str(resolved_packages.resolvable),
+          '(from: %s)' % resolved_packages.parent if resolved_packages.parent else '')
+    return ', '.join(
+        render_resolvable(resolved_packages) for resolved_packages in self.__tuples
+        if resolved_packages.resolvable.name == name)
+
   def _check(self):
     # Check whether or not the resolvables in this set are satisfiable, raise an exception if not.
-    for name, (resolvable, packages) in self._collapse().items():
-      if not packages:
-        raise self.Unsatisfiable('Could not satisfy all requirements for %s' % resolvable)
+    for name, resolved_packages in self._collapse().items():
+      if not resolved_packages.packages:
+        raise Unsatisfiable('Could not satisfy all requirements for %s:\n    %s' % (
+            resolved_packages.resolvable, self._synthesize_parents(name)))
 
-  def merge(self, resolvable, packages):
+  def merge(self, resolvable, packages, parent=None):
     """Add a resolvable and its resolved packages."""
-    self.__tuples.append((resolvable, OrderedSet(packages)))
+    self.__tuples.append(_ResolvedPackages(resolvable, OrderedSet(packages), parent))
     self._check()
 
   def get(self, name):
     """Get the set of compatible packages given a resolvable name."""
-    resolvable, packages = self._collapse().get(name, (None, OrderedSet()))
+    resolvable, packages, parent = self._collapse().get(name, _ResolvedPackages.empty())
     return packages
 
   def packages(self):
@@ -87,7 +110,7 @@ def packages(self):
 
   def extras(self, name):
     return set.union(
-        *[set(resolvable.extras()) for resolvable, _ in self.__tuples if resolvable.name == name])
+        *[set(tup.resolvable.extras()) for tup in self.__tuples if tup.resolvable.name == name])
 
 
 class Resolver(object):
@@ -127,7 +150,7 @@ def build(self, package, options):
     return dist
 
   def resolve(self, resolvables, resolvable_set=None):
-    resolvables = list(resolvables)
+    resolvables = [(resolvable, None) for resolvable in resolvables]
     resolvable_set = resolvable_set or _ResolvableSet()
     processed_resolvables = set()
     processed_packages = {}
@@ -135,14 +158,14 @@ def resolve(self, resolvables, resolvable_set=None):
 
     while resolvables:
       while resolvables:
-        resolvable = resolvables.pop(0)
+        resolvable, parent = resolvables.pop(0)
         if resolvable in processed_resolvables:
           continue
         packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name))
-        resolvable_set.merge(resolvable, packages)
+        resolvable_set.merge(resolvable, packages, parent)
         processed_resolvables.add(resolvable)
 
-      for resolvable, packages in resolvable_set.packages():
+      for resolvable, packages, parent in resolvable_set.packages():
         assert len(packages) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable
         package = next(iter(packages))
         if resolvable.name in processed_packages:
@@ -154,8 +177,9 @@ def resolve(self, resolvables, resolvable_set=None):
           distributions[package] = self.build(package, resolvable.options)
         distribution = distributions[package]
         processed_packages[resolvable.name] = package
+        new_parent = '%s->%s' % (parent, resolvable) if parent else str(resolvable)
         resolvables.extend(
-            ResolvableRequirement(req, resolvable.options) for req in
+            (ResolvableRequirement(req, resolvable.options), new_parent) for req in
             distribution.requires(extras=resolvable_set.extras(resolvable.name)))
 
     return list(distributions.values())


Test patch:
diff --git a/tests/test_resolver.py b/tests/test_resolver.py
--- a/tests/test_resolver.py
+++ b/tests/test_resolver.py
@@ -10,7 +10,7 @@
 from pex.fetcher import Fetcher
 from pex.package import EggPackage, SourcePackage
 from pex.resolvable import ResolvableRequirement
-from pex.resolver import _ResolvableSet, resolve
+from pex.resolver import Unsatisfiable, _ResolvableSet, resolve
 from pex.resolver_options import ResolverOptionsBuilder
 from pex.testing import make_sdist
 
@@ -43,7 +43,7 @@ def test_resolvable_set():
 
   rs.merge(rq, [source_pkg, binary_pkg])
   assert rs.get('foo') == set([source_pkg, binary_pkg])
-  assert rs.packages() == [(rq, set([source_pkg, binary_pkg]))]
+  assert rs.packages() == [(rq, set([source_pkg, binary_pkg]), None)]
 
   # test methods
   assert rs.extras('foo') == set(['ext'])
@@ -52,7 +52,7 @@ def test_resolvable_set():
   rs.merge(rq, [source_pkg])
   assert rs.get('foo') == set([source_pkg])
 
-  with pytest.raises(_ResolvableSet.Unsatisfiable):
+  with pytest.raises(Unsatisfiable):
     rs.merge(rq, [binary_pkg])