Repository: pantsbuild/pants

Issue:
Error for unhashable types used as Keys in the engine is unfortunate
Getting a `Key` for a `Value` in the engine requires hashing it, and when that fails, the error message is unfortunate (because we do not currently treat `Interns::put` or `externs::identify` as fallible operations):
```
From cffi callback <bound method _FFISpecification.extern_identify of <pants.engine.native._FFISpecification object at 0x10a100510>>:
Traceback (most recent call last):
  File "/Users/bescobar/workspace/otherpants/src/python/pants/engine/native.py", line 283, in extern_identify
    hash_ = hash(obj)
  File "/Users/bescobar/workspace/otherpants/src/python/pants/util/objects.py", line 107, in __hash__
    return super(DataType, self).__hash__()
TypeError: unhashable type: 'list'
thread 'tokio-runtime-worker-2' panicked at 'WithDeps(Inner(InnerEntry { params: {TypeId(140728165282080)}, rule: Task(Task { product: TypeConstraint(Key { id: 74, type_id: TypeId(140728165231312) }), clause: [Select { product: =Console }, Select { product: =TransitiveHydratedTargets }], gets: [Get { product: TypeConstraint(Key { id: 76, type_id: TypeId(140728165231312) }), subject: TypeId(140728202990560) }], func: Function(Key { id: 75, type_id: TypeId(4418260136) }), cacheable: false }) })) did not declare a dependency on JustGet(Get { product: TypeConstraint(Key { id: 76, type_id: TypeId(140728165231312) }), subject: TypeId(0) })', src/nodes.rs:839:13
```

----

While not _all_ `extern` methods should be fallible (although they can effectively all fail due to python exceptions), this one is probably a good candidate. Most of the objects that pass through the engine are `Values`, and relatively few need to be converted into `Keys`: essentially, only those used as `Params` (eg. as arguments to `Get(..)`). But introducing new calls to `Get(..)` is a much more common operation, and rendering a good error message in this case would be valuable.

To make this cheaper, and provide a better illusion of synchronous behaviour when you call `Get` with a bad argument, one approach would be to change the signature of [extern_generator_send](https://github.com/pantsbuild/pants/blob/00fd11701f9fb9bfcf3a5f9584c758d116d8d092/src/python/pants/engine/native.py#L418-L451) to directly return pre-identified handles (ie, including both the `Handle`, and an `Ident` as from `extern_identify`) when yielding a `Get`/`GetMulti`. This would make the `PyGeneratorResponse` struct more complicated though, because neither the `Break` nor `Throw` cases should identify their arguments.


Discussion:

Resolving patch:
diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py
--- a/src/python/pants/engine/native.py
+++ b/src/python/pants/engine/native.py
@@ -226,8 +226,9 @@ def wrapper(func):
 
 class _FFISpecification(object):
 
-  def __init__(self, ffi):
+  def __init__(self, ffi, lib):
     self._ffi = ffi
+    self._lib = lib
 
   @memoized_classproperty
   def _extern_fields(cls):
@@ -295,9 +296,7 @@ def extern_identify(self, context_handle, val):
     """
     c = self._ffi.from_handle(context_handle)
     obj = self._ffi.from_handle(val[0])
-    hash_ = hash(obj)
-    type_id = c.to_id(type(obj))
-    return (hash_, TypeId(type_id))
+    return c.identify(obj)
 
   @_extern_decl('_Bool', ['ExternContext*', 'Handle*', 'Handle*'])
   def extern_equals(self, context_handle, val1, val2):
@@ -419,36 +418,37 @@ def extern_create_exception(self, context_handle, msg_ptr, msg_len):
   def extern_generator_send(self, context_handle, func, arg):
     """Given a generator, send it the given value and return a response."""
     c = self._ffi.from_handle(context_handle)
+    response = self._ffi.new('PyGeneratorResponse*')
     try:
       res = c.from_value(func[0]).send(c.from_value(arg[0]))
       if isinstance(res, Get):
         # Get.
-        values = [res.subject]
-        products = [res.product]
-        tag = 2
+        response.tag = self._lib.Get
+        response.get = (
+            TypeId(c.to_id(res.product)),
+            c.to_value(res.subject),
+            c.identify(res.subject),
+          )
       elif type(res) in (tuple, list):
         # GetMulti.
-        values = [g.subject for g in res]
-        products = [g.product for g in res]
-        tag = 3
+        response.tag = self._lib.GetMulti
+        response.get_multi = (
+            c.type_ids_buf([TypeId(c.to_id(g.product)) for g in res]),
+            c.vals_buf([c.to_value(g.subject) for g in res]),
+            c.identities_buf([c.identify(g.subject) for g in res]),
+          )
       else:
         # Break.
-        values = [res]
-        products = []
-        tag = 0
+        response.tag = self._lib.Broke
+        response.broke = (c.to_value(res),)
     except Exception as e:
       # Throw.
+      response.tag = self._lib.Throw
       val = e
       val._formatted_exc = traceback.format_exc()
-      values = [val]
-      products = []
-      tag = 1
+      response.throw = (c.to_value(val),)
 
-    return (
-        tag,
-        c.vals_buf([c.to_value(v) for v in values]),
-        c.type_ids_buf([TypeId(c.to_id(t)) for t in products])
-      )
+    return response[0]
 
   @_extern_decl('PyResult', ['ExternContext*', 'Handle*', 'Handle**', 'uint64_t'])
   def extern_call(self, context_handle, func, args_ptr, args_len):
@@ -521,6 +521,10 @@ def vals_buf(self, vals):
     buf = self._ffi.new('Handle[]', vals)
     return (buf, len(vals), self.to_value(buf))
 
+  def identities_buf(self, idents):
+    buf = self._ffi.new('Ident[]', idents)
+    return (buf, len(idents), self.to_value(buf))
+
   def type_ids_buf(self, types):
     buf = self._ffi.new('TypeId[]', types)
     return (buf, len(types), self.to_value(buf))
@@ -545,6 +549,12 @@ def raise_or_return(self, pyresult):
   def drop_handles(self, handles):
     self._handles -= set(handles)
 
+  def identify(self, obj):
+    """Return an Ident-shaped tuple for the given object."""
+    hash_ = hash(obj)
+    type_id = self.to_id(type(obj))
+    return (hash_, TypeId(type_id))
+
   def to_id(self, typ):
     type_id = id(typ)
     self._types[type_id] = typ
@@ -589,14 +599,14 @@ def binary(self):
   @memoized_property
   def lib(self):
     """Load and return the native engine module."""
-    return self.ffi.dlopen(self.binary)
+    lib = self.ffi.dlopen(self.binary)
+    _FFISpecification(self.ffi, lib).register_cffi_externs()
+    return lib
 
   @memoized_property
   def ffi(self):
     """A CompiledCFFI handle as imported from the native engine python module."""
-    ffi = getattr(self._ffi_module, 'ffi')
-    _FFISpecification(ffi).register_cffi_externs()
-    return ffi
+    return getattr(self._ffi_module, 'ffi')
 
   @memoized_property
   def ffi_lib(self):


Test patch:
diff --git a/tests/python/pants_test/engine/test_scheduler.py b/tests/python/pants_test/engine/test_scheduler.py
--- a/tests/python/pants_test/engine/test_scheduler.py
+++ b/tests/python/pants_test/engine/test_scheduler.py
@@ -108,10 +108,18 @@ def __init__(self, inner):
 
 @rule(A, [TypeCheckFailWrapper])
 def a_typecheck_fail_test(wrapper):
-  # This `yield Get(A, B, ...)` will use the `nested_raise` rule defined above, but it won't get to
-  # the point of raising since the type check will fail at the Get.
-  supposedly_a = yield Get(A, B, wrapper.inner)
-  yield supposedly_a
+  # This `yield` would use the `nested_raise` rule, but it won't get to the point of raising since
+  # the type check will fail at the Get.
+  _ = yield Get(A, B, wrapper.inner) # noqa: F841
+  yield A()
+
+
+@rule(C, [TypeCheckFailWrapper])
+def c_unhashable(_):
+  # This `yield` would use the `nested_raise` rule, but it won't get to the point of raising since
+  # the hashability check will fail.
+  _ = yield Get(A, B, list()) # noqa: F841
+  yield C()
 
 
 @contextmanager
@@ -201,6 +209,7 @@ def rules(cls):
       RootRule(B),
       RootRule(TypeCheckFailWrapper),
       a_typecheck_fail_test,
+      c_unhashable,
       nested_raise,
     ]
 
@@ -213,6 +222,12 @@ def test_get_type_match_failure(self):
       # `a_typecheck_fail_test` above expects `wrapper.inner` to be a `B`.
       self.scheduler.product_request(A, [Params(TypeCheckFailWrapper(A()))])
 
+  def test_unhashable_failure(self):
+    """Test that unhashable Get(...) params result in a structured error."""
+    expected_msg = """TypeError: unhashable type: 'list'"""
+    with assert_execution_error(self, expected_msg):
+      self.scheduler.product_request(C, [Params(TypeCheckFailWrapper(A()))])
+
   def test_trace_includes_rule_exception_traceback(self):
     # Execute a request that will trigger the nested raise, and then directly inspect its trace.
     request = self.scheduler.execution_request([A], [B()])