Repository: pantsbuild/pants

Issue:
Python style `checker` pex requires runtime resolution of whl
Post #6618 a new `whl` is published, which is then resolved at runtime to generate a pex that can be subprocessed to check style.

This represents a slightly different model than what occurs currently with plugins, because plugins/backends can be loaded from within a deploy pex. But the `checker` whl/pex (currently) require independent resolution.

To simplify pants release processes, it would be great to allow the `checker` whl to be resolved off the python-path of the pants process itself.


Discussion:
Spawned from #5764
Can we leverage `PANTS_DEV` mode?

Resolving patch:
diff --git a/contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle/checkstyle.py b/contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle/checkstyle.py
--- a/contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle/checkstyle.py
+++ b/contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle/checkstyle.py
@@ -5,6 +5,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os
+import sys
 
 from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.subsystems.python_repos import PythonRepos
@@ -27,6 +28,7 @@
 from pex.interpreter import PythonInterpreter
 from pex.pex import PEX
 from pex.pex_builder import PEXBuilder
+from pkg_resources import DistributionNotFound, Requirement, WorkingSet
 
 from pants.contrib.python.checks.checker import checker
 from pants.contrib.python.checks.tasks.checkstyle.plugin_subsystem_base import \
@@ -56,6 +58,10 @@ def subsystem_dependencies(cls):
       # Needed implicitly by the pex_build_util functions we use.
       PythonSetup, PythonRepos)
 
+  @classmethod
+  def implementation_version(cls):
+    return super(Checkstyle, cls).implementation_version() + [('Checkstyle', 0)]
+
   @classmethod
   def prepare(cls, options, round_manager):
     round_manager.require_data(PythonInterpreter)
@@ -109,10 +115,21 @@ def checker_pex(self, interpreter):
                                                  req_libs=req_libs,
                                                  log=self.context.log)
           else:
-            pex_build_util.dump_requirements(builder,
-                                             interpreter=interpreter,
-                                             reqs=[PythonRequirement(self._CHECKER_REQ)],
-                                             log=self.context.log)
+            try:
+              # The checker is already on sys.path, eg: embedded in pants.pex.
+              working_set = WorkingSet(entries=sys.path)
+              for dist in working_set.resolve([Requirement.parse(self._CHECKER_REQ)]):
+                for req in dist.requires():
+                  builder.add_requirement(req)
+                builder.add_distribution(dist)
+              builder.add_requirement(self._CHECKER_REQ)
+            except DistributionNotFound:
+              # We need to resolve the checker from a local or remote distribution repo.
+              pex_build_util.dump_requirements(builder,
+                                               interpreter=interpreter,
+                                               reqs=[PythonRequirement(self._CHECKER_REQ)],
+                                               log=self.context.log)
+
           builder.set_entry_point(self._CHECKER_ENTRYPOINT)
           builder.freeze()
 


Test patch:
diff --git a/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/BUILD b/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/BUILD
--- a/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/BUILD
+++ b/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/BUILD
@@ -4,6 +4,8 @@
 python_tests(
   dependencies=[
     '3rdparty/python:future',
+    '3rdparty/python:parameterized',
+    '3rdparty/python:wheel',
     'contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle',
     'src/python/pants/backend/python/subsystems',
     'src/python/pants/backend/python/targets',
diff --git a/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/test_checker.py b/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/test_checker.py
--- a/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/test_checker.py
+++ b/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/test_checker.py
@@ -5,7 +5,9 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os
+import sys
 from builtins import str
+from contextlib import contextmanager
 from textwrap import dedent
 
 from pants.backend.python.subsystems.python_repos import PythonRepos
@@ -18,32 +20,49 @@
 from pants.util.dirutil import safe_mkdtemp, safe_rmtree
 from pants.util.process_handler import subprocess
 from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase
+from parameterized import parameterized
+from wheel.install import WheelFile
 
 from pants.contrib.python.checks.tasks.checkstyle.checkstyle import Checkstyle
 
 
-class CheckstyleTest(PythonTaskTestBase):
+CHECKER_RESOLVE_METHOD = [('sys.path', True), ('resolve', False)]
 
-  _distdir = None
-  _checker_dist = None
 
-  @classmethod
-  def setUpClass(cls):
-    cls._distdir = safe_mkdtemp()
+class CheckstyleTest(PythonTaskTestBase):
+
+  @staticmethod
+  def build_checker_wheel(root_dir):
     target = Checkstyle._CHECKER_ADDRESS_SPEC
     subprocess.check_call([os.path.join(get_buildroot(), 'pants'),
-                           '--pants-distdir={}'.format(cls._distdir),
+                           '--pants-distdir={}'.format(root_dir),
                            'setup-py',
                            '--run=bdist_wheel --universal',
                            target])
 
-    for root, dirs, files in os.walk(cls._distdir):
+    for root, _, files in os.walk(root_dir):
       for f in files:
         if f.endswith('.whl'):
-          cls._checker_dist = os.path.join(root, f)
-          break
-    if cls._checker_dist is None:
-      raise AssertionError('Failed to generate a wheel for {}'.format(target))
+          return os.path.join(root, f)
+
+    raise AssertionError('Failed to generate a wheel for {}'.format(target))
+
+  @staticmethod
+  def install_wheel(wheel, root_dir):
+    importable_path = os.path.join(root_dir, 'install', os.path.basename(wheel))
+    overrides = {path: root_dir for path in ('purelib', 'platlib', 'headers', 'scripts', 'data')}
+    WheelFile(wheel).install(force=True, overrides=overrides)
+    return importable_path
+
+  _distdir = None
+  _checker_dist = None
+  _checker_dist_importable_path = None
+
+  @classmethod
+  def setUpClass(cls):
+    cls._distdir = safe_mkdtemp()
+    cls._checker_dist = cls.build_checker_wheel(cls._distdir)
+    cls._checker_dist_importable_path = cls.install_wheel(cls._checker_dist, cls._distdir)
 
   @classmethod
   def tearDownClass(cls):
@@ -54,39 +73,56 @@ def tearDownClass(cls):
   def task_type(cls):
     return Checkstyle
 
-  def execute_task(self, target_roots=None):
-    with environment_as(PANTS_DEV=None, PEX_VERBOSE='9'):
+  @contextmanager
+  def resolve_configuration(self, resolve_local=False):
+    if resolve_local:
+      prior = sys.path[:]
+      sys.path.append(self._checker_dist_importable_path)
+      try:
+        yield
+      finally:
+        sys.path = prior
+    else:
       self.set_options_for_scope(PythonRepos.options_scope,
                                  repos=[os.path.dirname(self._checker_dist)])
       self.set_options_for_scope(PythonSetup.options_scope,
                                  resolver_allow_prereleases=True)
-      si_task_type = self.synthesize_task_subtype(SelectInterpreter, 'si_scope')
-      context = self.context(for_task_types=[si_task_type], target_roots=target_roots)
-      si_task_type(context, os.path.join(self.pants_workdir, 'si')).execute()
-      return self.create_task(context).execute()
-
-  def test_no_sources(self):
-    self.assertEqual(None, self.execute_task())
-
-  def test_pass(self):
+      yield
+
+  def execute_task(self, target_roots=None, resolve_local=False):
+    with self.resolve_configuration(resolve_local=resolve_local):
+      with environment_as(PANTS_DEV=None, PEX_VERBOSE='9'):
+        si_task_type = self.synthesize_task_subtype(SelectInterpreter, 'si_scope')
+        context = self.context(for_task_types=[si_task_type], target_roots=target_roots)
+        si_task_type(context, os.path.join(self.pants_workdir, 'si')).execute()
+        return self.create_task(context).execute()
+
+  @parameterized.expand(CHECKER_RESOLVE_METHOD)
+  def test_no_sources(self, unused_test_name, resolve_local):
+    self.assertEqual(None, self.execute_task(resolve_local=resolve_local))
+
+  @parameterized.expand(CHECKER_RESOLVE_METHOD)
+  def test_pass(self, unused_test_name, resolve_local):
     self.create_file('a/python/pass.py', contents=dedent("""
                        class UpperCase(object):
                          pass
                      """))
     target = self.make_target('a/python:pass', PythonLibrary, sources=['pass.py'])
-    self.assertEqual(0, self.execute_task(target_roots=[target]))
+    self.assertEqual(0, self.execute_task(target_roots=[target], resolve_local=resolve_local))
 
-  def test_failure(self):
+  @parameterized.expand(CHECKER_RESOLVE_METHOD)
+  def test_failure(self, unused_test_name, resolve_local):
     self.create_file('a/python/fail.py', contents=dedent("""
                         class lower_case(object):
                           pass
                        """))
     target = self.make_target('a/python:fail', PythonLibrary, sources=['fail.py'])
     with self.assertRaises(TaskError) as task_error:
-      self.execute_task(target_roots=[target])
+      self.execute_task(target_roots=[target], resolve_local=resolve_local)
     self.assertIn('1 Python Style issues found', str(task_error.exception))
 
-  def test_suppressed_file_passes(self):
+  @parameterized.expand(CHECKER_RESOLVE_METHOD)
+  def test_suppressed_file_passes(self, unused_test_name, resolve_local):
     self.create_file('a/python/fail.py', contents=dedent("""
                         class lower_case(object):
                           pass
@@ -95,21 +131,23 @@ class lower_case(object):
     a/python/fail\.py::variable-names"""))
     target = self.make_target('a/python:fail', PythonLibrary, sources=['fail.py'])
     self.set_options(suppress=suppression_file)
-    self.assertEqual(0, self.execute_task(target_roots=[target]))
+    self.assertEqual(0, self.execute_task(target_roots=[target], resolve_local=resolve_local))
 
-  def test_failure_fail_false(self):
+  @parameterized.expand(CHECKER_RESOLVE_METHOD)
+  def test_failure_fail_false(self, unused_test_name, resolve_local):
     self.create_file('a/python/fail.py', contents=dedent("""
                         class lower_case(object):
                           pass
                      """))
     target = self.make_target('a/python:fail', PythonLibrary, sources=['fail.py'])
     self.set_options(fail=False)
-    self.assertEqual(1, self.execute_task(target_roots=[target]))
+    self.assertEqual(1, self.execute_task(target_roots=[target], resolve_local=resolve_local))
 
-  def test_syntax_error(self):
+  @parameterized.expand(CHECKER_RESOLVE_METHOD)
+  def test_syntax_error(self, unused_test_name, resolve_local):
     self.create_file('a/python/error.py', contents=dedent("""
                          invalid python
                        """))
     target = self.make_target('a/python:error', PythonLibrary, sources=['error.py'])
     self.set_options(fail=False)
-    self.assertEqual(1, self.execute_task(target_roots=[target]))
+    self.assertEqual(1, self.execute_task(target_roots=[target], resolve_local=resolve_local))