Repository: pantsbuild/pants

Issue:
Prefer wheels during plugin install
Pants has sprouted a dependency on a package that is most easily installed via a `whl` on pypi (`openssl` for `requests[security]`).

But currently the plugin installation infrastructure does not enable usage of `WheelPackage` due to a defensive avoidance of assuming that `wheel` is installed in an environment that pants is being loaded from:
https://github.com/pantsbuild/pants/blob/e0d5108ff75a41421a11321c73c817332f0a1c86/src/python/pants/init/plugin_resolver.py#L79-L82

As far as @kwlzn or I can tell, `wheel` should always be on the PYTHONPATH, as pants itself depends on it: it's not clear that it actually needs to be "installed" on the PATH as well.

cc @jsirois for any context he might have on that comment.


Discussion:
I'm still not fully sure about the comment. That said, adding `WheelPackage` to the head of the `precedence` list (or deleting the custom `precedence` argument), leads to a new error - `pkg_resources` from `setuptools==30.0.0`, which Pants uses, does not know how to find plugins except if they reside in eggs. Newer versions `setuptools>=34.2.0` can do this however (http://setuptools.readthedocs.io/en/latest/history.html#v34-2-0).

Resolving patch:
diff --git a/src/python/pants/init/plugin_resolver.py b/src/python/pants/init/plugin_resolver.py
--- a/src/python/pants/init/plugin_resolver.py
+++ b/src/python/pants/init/plugin_resolver.py
@@ -8,17 +8,18 @@
 import hashlib
 import logging
 import os
+import site
 
 from pex import resolver
 from pex.base import requirement_is_exact
-from pex.package import EggPackage, SourcePackage
 from pkg_resources import working_set as global_working_set
 from pkg_resources import Requirement
+from wheel.install import WheelFile
 
 from pants.option.global_options import GlobalOptionsRegistrar
 from pants.python.python_repos import PythonRepos
 from pants.subsystem.subsystem import Subsystem
-from pants.util.dirutil import safe_open
+from pants.util.dirutil import safe_mkdir, safe_open
 from pants.util.memo import memoized_property
 from pants.version import PANTS_SEMVER
 
@@ -27,6 +28,26 @@
 
 
 class PluginResolver(object):
+  @staticmethod
+  def _is_wheel(path):
+    return os.path.isfile(path) and path.endswith('.whl')
+
+  @staticmethod
+  def _activate_wheel(wheel_path):
+    install_dir = '{}-install'.format(wheel_path)
+    safe_mkdir(install_dir, clean=True)
+    WheelFile(wheel_path).install(force=True,
+                                  overrides={
+                                    'purelib': install_dir,
+                                    'headers': os.path.join(install_dir, 'headers'),
+                                    'scripts': os.path.join(install_dir, 'bin'),
+                                    'platlib': install_dir,
+                                    'data': install_dir
+                                  })
+    # Activate any .pth files installed above.
+    site.addsitedir(install_dir)
+    return install_dir
+
   def __init__(self, options_bootstrapper):
     self._options_bootstrapper = options_bootstrapper
 
@@ -44,6 +65,8 @@ def resolve(self, working_set=None):
     working_set = working_set or global_working_set
     if self._plugin_requirements:
       for plugin_location in self._resolve_plugin_locations():
+        if self._is_wheel(plugin_location):
+          plugin_location = self._activate_wheel(plugin_location)
         working_set.add_entry(plugin_location)
     return working_set
 
@@ -76,15 +99,10 @@ def _resolve_exact_plugin_locations(self):
         yield plugin_location.strip()
 
   def _resolve_plugins(self):
-    # When bootstrapping plugins without the full pants python backend machinery in-play, we are not
-    # guaranteed a properly initialized interpreter with wheel support so we enforce eggs only for
-    # bdists with this custom precedence.
-    precedence = (EggPackage, SourcePackage)
     logger.info('Resolving new plugins...:\n  {}'.format('\n  '.join(self._plugin_requirements)))
     return resolver.resolve(self._plugin_requirements,
                             fetchers=self._python_repos.get_fetchers(),
                             context=self._python_repos.get_network_context(),
-                            precedence=precedence,
                             cache=self.plugin_cache_dir,
                             cache_ttl=10 * 365 * 24 * 60 * 60,  # Effectively never expire.
                             allow_prereleases=PANTS_SEMVER.is_prerelease)


Test patch:
diff --git a/tests/python/pants_test/init/BUILD b/tests/python/pants_test/init/BUILD
--- a/tests/python/pants_test/init/BUILD
+++ b/tests/python/pants_test/init/BUILD
@@ -5,6 +5,7 @@
 python_tests(
   dependencies = [
     '3rdparty/python:mock',
+    '3rdparty/python:parameterized',
     '3rdparty/python:pex',
     '3rdparty/python:setuptools',
     'src/python/pants/base:exceptions',
diff --git a/tests/python/pants_test/init/test_plugin_resolver.py b/tests/python/pants_test/init/test_plugin_resolver.py
--- a/tests/python/pants_test/init/test_plugin_resolver.py
+++ b/tests/python/pants_test/init/test_plugin_resolver.py
@@ -11,8 +11,9 @@
 from contextlib import contextmanager
 from textwrap import dedent
 
+from parameterized import parameterized
 from pex.crawler import Crawler
-from pex.installer import Packager
+from pex.installer import EggInstaller, Packager, WheelInstaller
 from pex.resolver import Unsatisfiable
 from pkg_resources import Requirement, WorkingSet
 
@@ -24,10 +25,12 @@
 
 req = Requirement.parse
 
+INSTALLERS = [('sdist', Packager), ('egg', EggInstaller), ('whl', WheelInstaller)]
+
 
 class PluginResolverTest(unittest.TestCase):
   @staticmethod
-  def create_plugin(distribution_repo_dir, plugin, version=None):
+  def create_plugin(distribution_repo_dir, plugin, version=None, packager_cls=None):
     with safe_open(os.path.join(distribution_repo_dir, plugin, 'setup.py'), 'w') as fp:
       fp.write(dedent("""
         from setuptools import setup
@@ -35,12 +38,13 @@ def create_plugin(distribution_repo_dir, plugin, version=None):
 
         setup(name="{plugin}", version="{version}")
       """).format(plugin=plugin, version=version or '0.0.0'))
-    packager = Packager(source_dir=os.path.join(distribution_repo_dir, plugin),
-                        install_dir=distribution_repo_dir)
+    packager_cls = packager_cls or Packager
+    packager = packager_cls(source_dir=os.path.join(distribution_repo_dir, plugin),
+                            install_dir=distribution_repo_dir)
     packager.run()
 
   @contextmanager
-  def plugin_resolution(self, chroot=None, plugins=None):
+  def plugin_resolution(self, chroot=None, plugins=None, packager_cls=None):
     @contextmanager
     def provide_chroot(existing):
       if existing:
@@ -64,7 +68,7 @@ def provide_chroot(existing):
             plugin, version = plugin
           plugin_list.append('{}=={}'.format(plugin, version) if version else plugin)
           if create_artifacts:
-            self.create_plugin(repo_dir, plugin, version)
+            self.create_plugin(repo_dir, plugin, version, packager_cls=packager_cls)
         env['PANTS_PLUGINS'] = '[{}]'.format(','.join(map(repr, plugin_list)))
 
       configpath = os.path.join(root_dir, 'pants.ini')
@@ -81,11 +85,10 @@ def test_no_plugins(self):
     with self.plugin_resolution() as (working_set, _, _, _):
       self.assertEqual([], working_set.entries)
 
-  def test_plugins(self):
-    with self.plugin_resolution(plugins=[('jake', '1.2.3'), 'jane']) as (working_set,
-                                                                         _,
-                                                                         _,
-                                                                         cache_dir):
+  @parameterized.expand(INSTALLERS)
+  def test_plugins(self, unused_test_name, packager_cls):
+    with self.plugin_resolution(plugins=[('jake', '1.2.3'), 'jane'],
+                                packager_cls=packager_cls) as (working_set, _, _, cache_dir):
       self.assertEqual(2, len(working_set.entries))
 
       dist = working_set.find(req('jake'))
@@ -98,8 +101,10 @@ def test_plugins(self):
       self.assertEqual(os.path.realpath(cache_dir),
                        os.path.realpath(os.path.dirname(dist.location)))
 
-  def test_exact_requirements(self):
-    with self.plugin_resolution(plugins=[('jake', '1.2.3'), ('jane', '3.4.5')]) as results:
+  @parameterized.expand(INSTALLERS)
+  def test_exact_requirements(self, unused_test_name, packager_cls):
+    with self.plugin_resolution(plugins=[('jake', '1.2.3'), ('jane', '3.4.5')],
+                                packager_cls=packager_cls) as results:
       working_set, chroot, repo_dir, cache_dir = results
 
       self.assertEqual(2, len(working_set.entries))
@@ -114,8 +119,10 @@ def test_exact_requirements(self):
 
         self.assertEqual(working_set.entries, working_set2.entries)
 
-  def test_inexact_requirements(self):
-    with self.plugin_resolution(plugins=[('jake', '1.2.3'), 'jane']) as results:
+  @parameterized.expand(INSTALLERS)
+  def test_inexact_requirements(self, unused_test_name, packager_cls):
+    with self.plugin_resolution(plugins=[('jake', '1.2.3'), 'jane'],
+                                packager_cls=packager_cls) as results:
       working_set, chroot, repo_dir, cache_dir = results
 
       self.assertEqual(2, len(working_set.entries))