Repository: pantsbuild/pants

Issue:
`PythonTarget.compatibility` can conflict with `PythonSetup.interpreter_constraints`
Right now Pants does not guard for this, it probably should.The universe of `PythonTarget.compatibility` in the BUILD graph should probably be a subset of the `PythonSetup.interpreter_constraints`.


Discussion:
Discovered working #5081
Probably related to fixes needed for #5081 

Resolving patch:
diff --git a/src/python/pants/backend/python/interpreter_cache.py b/src/python/pants/backend/python/interpreter_cache.py
--- a/src/python/pants/backend/python/interpreter_cache.py
+++ b/src/python/pants/backend/python/interpreter_cache.py
@@ -7,6 +7,7 @@
 import os
 import shutil
 from builtins import map, object, str
+from collections import defaultdict
 
 from pex.interpreter import PythonInterpreter
 from pex.package import EggPackage, Package, SourcePackage
@@ -74,29 +75,30 @@ def _cache_dir(self):
 
   def select_interpreter_for_targets(self, targets):
     """Pick an interpreter compatible with all the specified targets."""
-    tgts_with_compatibilities = []
+    tgts_by_compatibilities = defaultdict(list)
     filters = set()
     for target in targets:
-      if isinstance(target, PythonTarget) and target.compatibility:
-        tgts_with_compatibilities.append(target)
-        filters.update(target.compatibility)
+      if isinstance(target, PythonTarget):
+        c = self._python_setup.compatibility_or_constraints(target)
+        tgts_by_compatibilities[c].append(target)
+        filters.update(c)
 
     allowed_interpreters = set(self.setup(filters=filters))
 
     # Constrain allowed_interpreters based on each target's compatibility requirements.
-    for target in tgts_with_compatibilities:
-      compatible_with_target = set(self._matching(allowed_interpreters, target.compatibility))
+    for compatibility in tgts_by_compatibilities.keys():
+      compatible_with_target = set(self._matching(allowed_interpreters, compatibility))
       allowed_interpreters &= compatible_with_target
 
     if not allowed_interpreters:
       # Create a helpful error message.
-      unique_compatibilities = set(tuple(t.compatibility) for t in tgts_with_compatibilities)
+      unique_compatibilities = set(tuple(c) for c in tgts_by_compatibilities.keys())
       unique_compatibilities_strs = [','.join(x) for x in unique_compatibilities if x]
-      tgts_with_compatibilities_strs = [t.address.spec for t in tgts_with_compatibilities]
+      tgts_by_compatibilities_strs = [t[0].address.spec for t in tgts_by_compatibilities.values()]
       raise self.UnsatisfiableInterpreterConstraintsError(
         'Unable to detect a suitable interpreter for compatibilities: {} '
         '(Conflicting targets: {})'.format(' && '.join(sorted(unique_compatibilities_strs)),
-                                           ', '.join(tgts_with_compatibilities_strs)))
+                                           ', '.join(tgts_by_compatibilities_strs)))
     # Return the lowest compatible interpreter.
     return min(allowed_interpreters)
 
diff --git a/src/python/pants/backend/python/subsystems/python_setup.py b/src/python/pants/backend/python/subsystems/python_setup.py
--- a/src/python/pants/backend/python/subsystems/python_setup.py
+++ b/src/python/pants/backend/python/subsystems/python_setup.py
@@ -19,13 +19,6 @@ class PythonSetup(Subsystem):
   @classmethod
   def register_options(cls, register):
     super(PythonSetup, cls).register_options(register)
-    # Note: This replaces two options:
-    # A) The global --interpreter option in the old python tasks.
-    #    That flag is only relevant in the python backend, and should never have been
-    #    global to begin with.
-    # B) The --interpreter-requirement option above.  That flag merely served to set the
-    #    effective default for when no other constraints were set, so we might as well
-    #    roll that into the more general constraints.
     register('--interpreter-constraints', advanced=True, default=['CPython>=2.7,<3'], type=list,
              metavar='<requirement>',
              help="Constrain the selected Python interpreter.  Specify with requirement syntax, "
@@ -80,8 +73,7 @@ def register_options(cls, register):
 
   @property
   def interpreter_constraints(self):
-    return (self.get_options().interpreter_constraints or self.get_options().interpreter or
-            [self.get_options().interpreter_requirement or b''])
+    return tuple(self.get_options().interpreter_constraints)
 
   @property
   def interpreter_search_paths(self):
@@ -140,6 +132,10 @@ def artifact_cache_dir(self):
   def scratch_dir(self):
     return os.path.join(self.get_options().pants_workdir, *self.options_scope.split('.'))
 
+  def compatibility_or_constraints(self, target):
+    """Return either the compatibility of the given target, or the interpreter constraints."""
+    return tuple(target.compatibility or self.interpreter_constraints)
+
   def setuptools_requirement(self):
     return self._failsafe_parse('setuptools=={0}'.format(self.setuptools_version))
 
diff --git a/src/python/pants/backend/python/tasks/python_execution_task_base.py b/src/python/pants/backend/python/tasks/python_execution_task_base.py
--- a/src/python/pants/backend/python/tasks/python_execution_task_base.py
+++ b/src/python/pants/backend/python/tasks/python_execution_task_base.py
@@ -10,6 +10,7 @@
 from pex.interpreter import PythonInterpreter
 from pex.pex import PEX
 
+from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_target import PythonTarget
@@ -70,6 +71,10 @@ def add_to(self, builder):
         add = builder.add_source if self.path.endswith('.py') else builder.add_resource
         add(fp.name, self.path)
 
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(PythonExecutionTaskBase, cls).subsystem_dependencies() + (PythonSetup,)
+
   def extra_files(self):
     """Override to provide extra files needed for execution.
 
@@ -112,7 +117,7 @@ def create_pex(self, pex_info=None):
           # in the target set's dependency closure.
           pexes = [extra_requirements_pex] + pexes
         constraints = {constraint for rt in relevant_targets if is_python_target(rt)
-                       for constraint in rt.compatibility}
+                       for constraint in PythonSetup.global_instance().compatibility_or_constraints(rt)}
 
         with self.merged_pex(path, pex_info, interpreter, pexes, constraints) as builder:
           for extra_file in self.extra_files():


Test patch:
diff --git a/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py b/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py
--- a/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py
+++ b/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py
@@ -126,25 +126,30 @@ def test_pants_binary_interpreter_selection_with_pexrc(self):
     os.remove(py3_pex)
 
   def test_target_constraints_with_no_sources(self):
+    if self.skip_if_no_python('3'):
+      return
+
     with temporary_dir() as interpreters_cache:
+      pants_ini_config = {
+          'python-setup': {
+            'interpreter_cache_dir': interpreters_cache,
+            'interpreter_constraints': ['CPython>3'],
+          }
+        }
       # Run task.
-      py3 = '3'
-      if not self.skip_if_no_python(py3):
-        pants_ini_config = {'python-setup': {'interpreter_cache_dir': interpreters_cache}}
-        pants_run_3 = self.run_pants(
-          command=['run', '{}:test_bin'.format(os.path.join(self.testproject, 'test_target_with_no_sources'))],
-          config=pants_ini_config
-        )
-        self.assert_success(pants_run_3)
-        self.assertIn('python3', pants_run_3.stdout_data)
+      pants_run = self.run_pants(
+        command=['run', '{}:test_bin'.format(os.path.join(self.testproject, 'test_target_with_no_sources'))],
+        config=pants_ini_config
+      )
+      self.assert_success(pants_run)
+      self.assertIn('python3', pants_run.stdout_data)
 
       # Binary task.
-      pants_ini_config = {'python-setup': {'interpreter_cache_dir': interpreters_cache}}
-      pants_run_27 = self.run_pants(
+      pants_run = self.run_pants(
         command=['binary', '{}:test_bin'.format(os.path.join(self.testproject, 'test_target_with_no_sources'))],
         config=pants_ini_config
       )
-      self.assert_success(pants_run_27)
+      self.assert_success(pants_run)
 
     # Ensure proper interpreter constraints were passed to built pexes.
     py2_pex = os.path.join(os.getcwd(), 'dist', 'test_bin.pex')
diff --git a/tests/python/pants_test/python/test_interpreter_selection_integration.py b/tests/python/pants_test/python/test_interpreter_selection_integration.py
--- a/tests/python/pants_test/python/test_interpreter_selection_integration.py
+++ b/tests/python/pants_test/python/test_interpreter_selection_integration.py
@@ -14,11 +14,29 @@
 class InterpreterSelectionIntegrationTest(PantsRunIntegrationTest):
   testproject = 'testprojects/src/python/interpreter_selection'
 
-  def test_conflict(self):
+  def test_conflict_via_compatibility(self):
+    # Tests that targets with compatibility conflicts collide.
     binary_target = '{}:deliberately_conficting_compatibility'.format(self.testproject)
     pants_run = self._build_pex(binary_target)
     self.assert_failure(pants_run,
                         'Unexpected successful build of {binary}.'.format(binary=binary_target))
+    self.assertIn('Unable to detect a suitable interpreter for compatibilities',
+                  pants_run.stdout_data)
+
+  def test_conflict_via_config(self):
+    # Tests that targets with compatibility conflict with targets with default compatibility.
+    # NB: Passes empty `args` to avoid having the default CLI args override the config.
+    config = {
+        'python-setup': {
+          'interpreter_constraints': ['CPython<2.7'],
+        }
+      }
+    binary_target = '{}:echo_interpreter_version'.format(self.testproject)
+    pants_run = self._build_pex(binary_target, config=config, args=[])
+    self.assert_failure(pants_run,
+                        'Unexpected successful build of {binary}.'.format(binary=binary_target))
+    self.assertIn('Unable to detect a suitable interpreter for compatibilities',
+                  pants_run.stdout_data)
 
   def test_select_3(self):
     self._maybe_test_version('3')
@@ -56,10 +74,11 @@ def _echo_version(self, version):
       (stdout_data, _) = proc.communicate()
       return stdout_data
 
-  def _build_pex(self, binary_target, config=None):
-    # Avoid some known-to-choke-on interpreters.
-    command = ['binary',
-               binary_target,
-               '--python-setup-interpreter-constraints=CPython>=2.7,<3',
-               '--python-setup-interpreter-constraints=CPython>=3.3']
+  def _build_pex(self, binary_target, config=None, args=None):
+    # By default, Avoid some known-to-choke-on interpreters.
+    args = list(args) if args is not None else [
+          '--python-setup-interpreter-constraints=CPython>=2.7,<3',
+          '--python-setup-interpreter-constraints=CPython>=3.3',
+        ]
+    command = ['binary', binary_target] + args
     return self.run_pants(command=command, config=config)