Repository: pantsbuild/pants

Issue:
Convert `invoke_runnable` to a general `call` and use to harden externs
The `invoke_runnable` extern is shaped like python `__call__`, and could replace a few other externs (`store_snapshot`, etc) with some minor tweaks.

Additionally, `project` and `project_multi` both invoke user code in many cases, meaning that they should really result in either success or failure like `invoke_runnable` does. Doing that would force failures to be handled, and implementing `project`/`project_multi` in terms of `invoke_runnable` would help with this.


Discussion:

Resolving patch:
diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py
--- a/src/python/pants/engine/native.py
+++ b/src/python/pants/engine/native.py
@@ -79,10 +79,9 @@
 } BufferBuffer;
 
 typedef struct {
-  Value  value;
   _Bool  is_throw;
-  Buffer traceback;
-} RunnableComplete;
+  Value  value;
+} PyResult;
 
 typedef void ExternContext;
 
@@ -103,7 +102,8 @@
 typedef ValueBuffer      (*extern_ptr_project_multi)(ExternContext*, Value*, uint8_t*, uint64_t);
 typedef Value            (*extern_ptr_project_ignoring_type)(ExternContext*, Value*, uint8_t*, uint64_t);
 typedef Value            (*extern_ptr_create_exception)(ExternContext*, uint8_t*, uint64_t);
-typedef RunnableComplete (*extern_ptr_invoke_runnable)(ExternContext*, Value*, Value*, uint64_t, _Bool);
+typedef PyResult         (*extern_ptr_call)(ExternContext*, Value*, Value*, uint64_t);
+typedef PyResult         (*extern_ptr_eval)(ExternContext*, uint8_t*, uint64_t);
 
 typedef void Tasks;
 typedef void Scheduler;
@@ -127,6 +127,8 @@
 CFFI_HEADERS = '''
 void externs_set(ExternContext*,
                  extern_ptr_log,
+                 extern_ptr_call,
+                 extern_ptr_eval,
                  extern_ptr_key_for,
                  extern_ptr_val_for,
                  extern_ptr_clone_val,
@@ -142,7 +144,6 @@
                  extern_ptr_project_ignoring_type,
                  extern_ptr_project_multi,
                  extern_ptr_create_exception,
-                 extern_ptr_invoke_runnable,
                  TypeId);
 
 Tasks* tasks_create(void);
@@ -196,7 +197,7 @@
 void graph_visualize(Scheduler*, ExecutionRequest*, char*);
 void graph_trace(Scheduler*, ExecutionRequest*, char*);
 
-void execution_add_root_select(Scheduler*, ExecutionRequest*, Key, TypeConstraint);
+PyResult  execution_add_root_select(Scheduler*, ExecutionRequest*, Key, TypeConstraint);
 RawNodes* execution_execute(Scheduler*, ExecutionRequest*);
 
 Value validator_run(Scheduler*);
@@ -216,6 +217,8 @@
 CFFI_EXTERNS = '''
 extern "Python" {
   void             extern_log(ExternContext*, uint8_t, uint8_t*, uint64_t);
+  PyResult         extern_call(ExternContext*, Value*, Value*, uint64_t);
+  PyResult         extern_eval(ExternContext*, uint8_t*, uint64_t);
   Key              extern_key_for(ExternContext*, Value*);
   Value            extern_val_for(ExternContext*, Key*);
   Value            extern_clone_val(ExternContext*, Value*);
@@ -231,7 +234,6 @@
   Value            extern_project_ignoring_type(ExternContext*, Value*, uint8_t*, uint64_t);
   ValueBuffer      extern_project_multi(ExternContext*, Value*, uint8_t*, uint64_t);
   Value            extern_create_exception(ExternContext*, uint8_t*, uint64_t);
-  RunnableComplete extern_invoke_runnable(ExternContext*, Value*, Value*, uint64_t, _Bool);
 }
 '''
 
@@ -283,6 +285,17 @@ def _initialize_externs(ffi):
   def to_py_str(msg_ptr, msg_len):
     return bytes(ffi.buffer(msg_ptr, msg_len)).decode('utf-8')
 
+  def call(c, func, args):
+    try:
+      val = func(*args)
+      is_throw = False
+    except Exception as e:
+      val = e
+      is_throw = True
+      e._formatted_exc = traceback.format_exc()
+
+    return PyResult(is_throw, c.to_value(val))
+
   @ffi.def_extern()
   def extern_log(context_handle, level, msg_ptr, msg_len):
     """Given a log level and utf8 message string, log it."""
@@ -416,22 +429,18 @@ def extern_create_exception(context_handle, msg_ptr, msg_len):
     return c.to_value(Exception(msg))
 
   @ffi.def_extern()
-  def extern_invoke_runnable(context_handle, func, args_ptr, args_len, cacheable):
-    """Given a destructured rawRunnable, run it."""
+  def extern_call(context_handle, func, args_ptr, args_len):
+    """Given a callable, call it."""
     c = ffi.from_handle(context_handle)
     runnable = c.from_value(func)
     args = tuple(c.from_value(arg) for arg in ffi.unpack(args_ptr, args_len))
+    return call(c, runnable, args)
 
-    try:
-      val = runnable(*args)
-      is_throw = False
-      traceback_str = ''
-    except Exception as e:
-      val = e
-      is_throw = True
-      traceback_str = traceback.format_exc()
-
-    return RunnableComplete(c.to_value(val), is_throw, c.utf8_buf(traceback_str))
+  @ffi.def_extern()
+  def extern_eval(context_handle, python_code_str_ptr, python_code_str_len):
+    """Given an evalable string, eval it and return a Value for its result."""
+    c = ffi.from_handle(context_handle)
+    return call(c, eval, [to_py_str(python_code_str_ptr, python_code_str_len)])
 
 
 class Value(datatype('Value', ['handle'])):
@@ -454,7 +463,7 @@ class TypeId(datatype('TypeId', ['id_'])):
   """Corresponds to the native object of the same name."""
 
 
-class RunnableComplete(datatype('RunnableComplete', ['value', 'is_throw', 'traceback'])):
+class PyResult(datatype('PyResult', ['is_throw', 'value'])):
   """Corresponds to the native object of the same name."""
 
 
@@ -632,6 +641,8 @@ def init_externs():
       context = ExternContext(self.ffi)
       self.lib.externs_set(context.handle,
                            self.ffi_lib.extern_log,
+                           self.ffi_lib.extern_call,
+                           self.ffi_lib.extern_eval,
                            self.ffi_lib.extern_key_for,
                            self.ffi_lib.extern_val_for,
                            self.ffi_lib.extern_clone_val,
@@ -647,7 +658,6 @@ def init_externs():
                            self.ffi_lib.extern_project_ignoring_type,
                            self.ffi_lib.extern_project_multi,
                            self.ffi_lib.extern_create_exception,
-                           self.ffi_lib.extern_invoke_runnable,
                            TypeId(context.to_id(str)))
       return context
 
diff --git a/src/python/pants/engine/scheduler.py b/src/python/pants/engine/scheduler.py
--- a/src/python/pants/engine/scheduler.py
+++ b/src/python/pants/engine/scheduler.py
@@ -265,10 +265,12 @@ def exec_reset(self):
     self._native.lib.execution_reset(self._scheduler)
 
   def add_root_selection(self, execution_request, subject, product):
-    self._native.lib.execution_add_root_select(self._scheduler,
-                                               execution_request,
-                                               self._to_key(subject),
-                                               self._to_constraint(product))
+    res = self._native.lib.execution_add_root_select(self._scheduler,
+                                                     execution_request,
+                                                     self._to_key(subject),
+                                                     self._to_constraint(product))
+    if res.is_throw:
+      raise self._from_value(res.value)
 
   def visualize_to_dir(self):
     return self._native.visualize_to_dir


Test patch:
diff --git a/tests/python/pants_test/engine/test_engine.py b/tests/python/pants_test/engine/test_engine.py
--- a/tests/python/pants_test/engine/test_engine.py
+++ b/tests/python/pants_test/engine/test_engine.py
@@ -125,8 +125,8 @@ def test_include_trace_error_raises_error_with_trace(self):
         Computing Task(<function nested_raise at 0xEEEEEEEEE>, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A)
           Throw(An exception for B)
             Traceback (most recent call last):
-              File LOCATION-INFO, in extern_invoke_runnable
-                val = runnable(*args)
+              File LOCATION-INFO, in call
+                val = func(*args)
               File LOCATION-INFO, in nested_raise
                 fn_raises(x)
               File LOCATION-INFO, in fn_raises
@@ -156,8 +156,8 @@ def test_trace_does_not_include_cancellations(self):
           Computing Task(<function nested_raise at 0xEEEEEEEEE>, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =C)
             Throw(An exception for B)
               Traceback (most recent call last):
-                File LOCATION-INFO, in extern_invoke_runnable
-                  val = runnable(*args)
+                File LOCATION-INFO, in call
+                  val = func(*args)
                 File LOCATION-INFO, in nested_raise
                   fn_raises(x)
                 File LOCATION-INFO, in fn_raises
@@ -165,3 +165,16 @@ def test_trace_does_not_include_cancellations(self):
               Exception: An exception for B
       ''').lstrip()+'\n',
       remove_locations_from_traceback(str(cm.exception)))
+
+  def test_illegal_root_selection(self):
+    rules = [
+      RootRule(B),
+    ]
+
+    scheduler = self.scheduler(rules, include_trace_on_error=False)
+
+    # No rules are available to compute A.
+    with self.assertRaises(Exception) as cm:
+      list(scheduler.product_request(A, subjects=[(B())]))
+
+    self.assert_equal_with_printing('No installed rules can satisfy Select(A) for a root subject of type B.', str(cm.exception))
diff --git a/tests/python/pants_test/engine/test_scheduler.py b/tests/python/pants_test/engine/test_scheduler.py
--- a/tests/python/pants_test/engine/test_scheduler.py
+++ b/tests/python/pants_test/engine/test_scheduler.py
@@ -269,8 +269,8 @@ def test_trace_includes_rule_exception_traceback(self):
                        Computing Task(<function nested_raise at 0xEEEEEEEEE>, <pants_test.engine.test_scheduler.B object at 0xEEEEEEEEE>, =A)
                          Throw(An exception for B)
                            Traceback (most recent call last):
-                             File LOCATION-INFO, in extern_invoke_runnable
-                               val = runnable(*args)
+                             File LOCATION-INFO, in call
+                               val = func(*args)
                              File LOCATION-INFO, in nested_raise
                                fn_raises(x)
                              File LOCATION-INFO, in fn_raises