Repository: pantsbuild/pants

Issue:
ConanPrep runs an expensive bootstrap even when not needed
This behavior is inherited from the PythonToolPrepBase base Task:
https://github.com/pantsbuild/pants/blob/c0db03f4b1c0acd1a2bb2ccff7ba0405ea1f26a6/src/python/pants/backend/python/tasks/python_tool_prep_base.py#L75-L82

Instead of pinning the fix to rules, it could also be easily whack-a-moled with an optionally implementable predicate which we could implement for the expensive PythonToolPrepBase instances - ConanPrep being the most notable.


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/native/tasks/conan_prep.py b/src/python/pants/backend/native/tasks/conan_prep.py
--- a/src/python/pants/backend/native/tasks/conan_prep.py
+++ b/src/python/pants/backend/native/tasks/conan_prep.py
@@ -5,6 +5,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from pants.backend.native.subsystems.conan import Conan
+from pants.backend.native.targets.external_native_library import ExternalNativeLibrary
 from pants.backend.python.tasks.python_tool_prep_base import PythonToolInstance, PythonToolPrepBase
 
 
@@ -15,3 +16,6 @@ class ConanInstance(PythonToolInstance):
 class ConanPrep(PythonToolPrepBase):
   tool_subsystem_cls = Conan
   tool_instance_cls = ConanInstance
+
+  def will_be_invoked(self):
+    return any(self.get_targets(lambda t: isinstance(t, ExternalNativeLibrary)))
diff --git a/src/python/pants/backend/python/tasks/python_tool_prep_base.py b/src/python/pants/backend/python/tasks/python_tool_prep_base.py
--- a/src/python/pants/backend/python/tasks/python_tool_prep_base.py
+++ b/src/python/pants/backend/python/tasks/python_tool_prep_base.py
@@ -72,12 +72,14 @@ def run(self, *args, **kwargs):
       return cmdline, exit_code
 
 
-# TODO: This python tool setup ends up eagerly generating each pex for each task in every goal which
-# is transitively required by the command-line goals, even for tasks which no-op. This requires each
-# pex for each relevant python tool to be buildable on the current host, even if it may never be
-# intended to be invoked. Especially given the existing clear separation of concerns into
-# PythonToolBase/PythonToolInstance/PythonToolPrepBase, this seems like an extremely ripe use case
-# for some v2 rules for free caching and no-op when not required for the command-line goals.
+# TODO: If `.will_be_invoked()` is not overridden, this python tool setup ends up eagerly generating
+# each pex for each task in every goal which is transitively required by the command-line goals,
+# even for tasks which no-op. This requires each pex for each relevant python tool to be buildable
+# on the current host, even if it may never be intended to be invoked, unless the prep task is able
+# to know whether it will be invoked in advance. Especially given the existing clear separation of
+# concerns into PythonToolBase/PythonToolInstance/PythonToolPrepBase, this seems like an extremely
+# ripe use case for some v2 rules for free caching and no-op when not required for the command-line
+# goals.
 class PythonToolPrepBase(Task):
   """Base class for tasks that resolve a python tool to be invoked out-of-process."""
 
@@ -127,7 +129,14 @@ def _generate_fingerprinted_pex_path(self, tool_subsystem, interpreter):
       '{}-{}.pex'.format(tool_subsystem.options_scope, specs_fingerprint),
     )
 
+  def will_be_invoked(self):
+    """Predicate which can be overridden to allow tool creation to no-op when not needed."""
+    return True
+
   def execute(self):
+    if not self.will_be_invoked():
+      return
+
     tool_subsystem = self.tool_subsystem_cls.scoped_instance(self)
 
     interpreter_cache = PythonInterpreterCache.global_instance()


Test patch:
diff --git a/tests/python/pants_test/backend/native/tasks/test_conan_fetch.py b/tests/python/pants_test/backend/native/tasks/test_conan_fetch.py
--- a/tests/python/pants_test/backend/native/tasks/test_conan_fetch.py
+++ b/tests/python/pants_test/backend/native/tasks/test_conan_fetch.py
@@ -6,6 +6,7 @@
 
 import os
 
+from pants.backend.native.targets.external_native_library import ExternalNativeLibrary
 from pants.backend.native.tasks.conan_fetch import ConanFetch
 from pants.backend.native.tasks.conan_prep import ConanPrep
 from pants_test.task_test_base import TaskTestBase
@@ -17,10 +18,23 @@ class ConanFetchTest(TaskTestBase):
   def task_type(cls):
     return ConanFetch
 
+  def test_conan_pex_noop(self):
+    """Test that the conan pex is not generated if there are no conan libraries to fetch."""
+    conan_prep_task_type = self.synthesize_task_subtype(ConanPrep, 'conan_prep_scope')
+    context = self.context(for_task_types=[conan_prep_task_type])
+    conan_prep = conan_prep_task_type(context, os.path.join(self.pants_workdir, 'conan_prep'))
+    conan_prep.execute()
+    self.assertIsNone(context.products.get_data(ConanPrep.tool_instance_cls))
+
   def test_rewrites_remotes_according_to_options(self):
     self.set_options(conan_remotes={'pants-conan': 'https://conan.bintray.com'})
     conan_prep_task_type = self.synthesize_task_subtype(ConanPrep, 'conan_prep_scope')
-    context = self.context(for_task_types=[conan_prep_task_type])
+    # We need at least one library to resolve here so that the conan pex is generated.
+    dummy_target = self.make_target(spec='//:dummy-conan-3rdparty-lib',
+                                    target_type=ExternalNativeLibrary,
+                                    packages=[])
+    context = self.context(for_task_types=[conan_prep_task_type],
+                           target_roots=[dummy_target])
     conan_prep = conan_prep_task_type(context, os.path.join(self.pants_workdir, 'conan_prep'))
     conan_fetch = self.create_task(context, os.path.join(self.pants_workdir, 'conan_fetch'))
     conan_prep.execute()
diff --git a/tests/python/pants_test/backend/python/tasks/test_python_tool.py b/tests/python/pants_test/backend/python/tasks/test_python_tool.py
--- a/tests/python/pants_test/backend/python/tasks/test_python_tool.py
+++ b/tests/python/pants_test/backend/python/tasks/test_python_tool.py
@@ -22,6 +22,11 @@ class Tool(PythonToolBase):
   ]
   default_entry_point = 'pex.bin.pex:main'
 
+  @classmethod
+  def register_options(cls, register):
+    super(Tool, cls).register_options(register)
+    register('--needs-to-be-invoked-for-some-reason', type=bool, default=True)
+
 
 class ToolInstance(PythonToolInstance):
   pass
@@ -32,6 +37,9 @@ class ToolPrep(PythonToolPrepBase):
   tool_subsystem_cls = Tool
   tool_instance_cls = ToolInstance
 
+  def will_be_invoked(self):
+    return Tool.scoped_instance(self).get_options().needs_to_be_invoked_for_some_reason
+
 
 class ToolTask(Task):
   options_scope = 'tool-task'
@@ -70,9 +78,10 @@ def _assert_tool_execution_for_python_version(self, use_py3=True):
       tool_prep_task = tool_prep_type(context, os.path.join(
         self.pants_workdir, 'tp_py{}'.format(scope_string)))
       tool_prep_task.execute()
+      pex_tool = context.products.get_data(ToolPrep.tool_instance_cls)
+      self.assertIsNotNone(pex_tool)
       # Check that the tool can be created and executed successfully.
       self.create_task(context).execute()
-      pex_tool = context.products.get_data(ToolPrep.tool_instance_cls)
       # Check that our pex tool wrapper was constructed with the expected interpreter.
       self.assertTrue(pex_tool.interpreter.identity.matches(constraint_string))
       return pex_tool
@@ -86,3 +95,14 @@ def test_tool_execution(self):
     py2_pex_tool_path = py2_pex_tool.pex.path()
     self.assertTrue(os.path.isdir(py2_pex_tool_path))
     self.assertNotEqual(py3_pex_tool_path, py2_pex_tool_path)
+
+  def test_tool_noop(self):
+    tool_prep_type = self.synthesize_task_subtype(ToolPrep, 'tool_prep')
+    context = self.context(for_task_types=[tool_prep_type], for_subsystems=[Tool], options={
+      'test-tool': {
+        'needs_to_be_invoked_for_some_reason': False,
+      },
+    })
+    tool_prep_task = tool_prep_type(context, os.path.join(self.pants_workdir, 'tool_prep_dir'))
+    tool_prep_task.execute()
+    self.assertIsNone(context.products.get_data(ToolPrep.tool_instance_cls))