Repository: pantsbuild/pants

Issue:
Consider switching to "new style" cffi callbacks
To avoid needing a compile step, we currently use "old style" cffi callbacks. The new style are purportedly faster and less error prone, but they require an additional build step, and so we would want to confirm the speedup before switching.


Discussion:
It looks like switching to the new callbacks makes the entire execution about 10% faster overall (meaning the individual CFFI calls were probably 50%ish faster). And the profile got about 10x more usable, which is great.

Additionally, it looks like it would probably replace the way we ship the native engine binary: instead, you'd have two `*.so` files: one for the cffi generated extern stub python module, and one for the rust binary. Loading the generated stub module `*.so` triggers loading of the second `*.so` in the same directory.

The only issue (and it seems like a significant one to me) is that it links against python. It feels like that would complicate things. If that is surmountable, then I think we should absolutely switch.

cc @kwlzn
I've posted some partial work on this one [here](https://github.com/pantsbuild/pants/compare/74ea25fa9966e63a2fbd38395edf225efba46ba0...twitter:stuhood/engine-cffi-new-callbacks)... it's working, but manually integrated in the form of two checked-in `*.so` files.
cc @kwlzn 

Resolving patch:
diff --git a/build-support/native-engine/bootstrap_cffi.py b/build-support/native-engine/bootstrap_cffi.py
new file mode 100644
--- /dev/null
+++ b/build-support/native-engine/bootstrap_cffi.py
@@ -0,0 +1,19 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import sys
+
+from pants.engine.subsystem.native import bootstrap_c_source
+
+
+if __name__ == '__main__':
+  try:
+    output_dir = sys.argv[1]
+  except Exception:
+    print('usage: {} <output dir>'.format(sys.argv[0]))
+
+  bootstrap_c_source(output_dir)
diff --git a/src/python/pants/engine/subsystem/native.py b/src/python/pants/engine/subsystem/native.py
--- a/src/python/pants/engine/subsystem/native.py
+++ b/src/python/pants/engine/subsystem/native.py
@@ -5,17 +5,22 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import importlib
 import logging
+import os
+import sys
+import sysconfig
 import threading
 
+import cffi
 import pkg_resources
 import six
-from cffi import FFI
 
 from pants.binaries.binary_util import BinaryUtil
 from pants.engine.storage import Storage
 from pants.option.custom_types import dir_option
 from pants.subsystem.subsystem import Subsystem
+from pants.util.dirutil import safe_mkdir
 from pants.util.memo import memoized_property
 from pants.util.objects import datatype
 
@@ -23,348 +28,387 @@
 logger = logging.getLogger(__name__)
 
 
-_FFI = FFI()
-_FFI.cdef(
-    '''
-    typedef uint64_t   Id;
-    typedef void*      Handle;
-
-    typedef struct {
-      Id id_;
-    } TypeId;
-
-    typedef struct {
-      Id id_;
-    } TypeConstraint;
-
-    typedef struct {
-      Id id_;
-    } Function;
-
-    typedef struct {
-      Handle   handle;
-    } Value;
-
-    typedef struct {
-      Id       id_;
-      TypeId   type_id;
-    } Key;
-
-    typedef struct {
-      uint8_t*  bytes_ptr;
-      uint64_t  bytes_len;
-      Value     handle_;
-    } Buffer;
-
-    typedef struct {
-      Value*     values_ptr;
-      uint64_t   values_len;
-      Value      handle_;
-    } ValueBuffer;
-
-    typedef struct {
-      TypeId*     ids_ptr;
-      uint64_t    ids_len;
-      Value       handle_;
-    } TypeIdBuffer;
-
-    typedef struct {
-      Buffer*     bufs_ptr;
-      uint64_t    bufs_len;
-      Value       handle_;
-    } BufferBuffer;
-
-    typedef struct {
-      Value  value;
-      bool   is_throw;
-    } RunnableComplete;
-
-    typedef uint64_t EntryId;
-
-    typedef void ExternContext;
-
-    // On the rust side the integration is defined in externs.rs
-    typedef void             (*extern_log)(ExternContext*, uint8_t, uint8_t*, uint64_t);
-    typedef Key              (*extern_key_for)(ExternContext*, Value*);
-    typedef Value            (*extern_val_for)(ExternContext*, Key*);
-    typedef Value            (*extern_clone_val)(ExternContext*, Value*);
-    typedef void             (*extern_drop_handles)(ExternContext*, Handle*, uint64_t);
-    typedef Buffer           (*extern_id_to_str)(ExternContext*, Id);
-    typedef Buffer           (*extern_val_to_str)(ExternContext*, Value*);
-    typedef bool             (*extern_satisfied_by)(ExternContext*, TypeConstraint*, Value*);
-    typedef bool             (*extern_satisfied_by_type)(ExternContext*, TypeConstraint*, TypeId*);
-    typedef Value            (*extern_store_list)(ExternContext*, Value**, uint64_t, bool);
-    typedef Value            (*extern_store_bytes)(ExternContext*, uint8_t*, uint64_t);
-    typedef Value            (*extern_project)(ExternContext*, Value*, uint8_t*, uint64_t, TypeId*);
-    typedef ValueBuffer      (*extern_project_multi)(ExternContext*, Value*, uint8_t*, uint64_t);
-    typedef Value            (*extern_project_ignoring_type)(ExternContext*, Value*, uint8_t*, uint64_t);
-    typedef Value            (*extern_create_exception)(ExternContext*, uint8_t*, uint64_t);
-    typedef RunnableComplete (*extern_invoke_runnable)(ExternContext*, Value*, Value*, uint64_t, bool);
-
-    typedef void RawScheduler;
-
-    typedef struct {
-      uint64_t runnable_count;
-      uint64_t scheduling_iterations;
-    } ExecutionStat;
-
-    typedef struct {
-      Key             subject;
-      TypeConstraint  product;
-      uint8_t         state_tag;
-      Value           state_value;
-    } RawNode;
-
-    typedef struct {
-      RawNode*  nodes_ptr;
-      uint64_t  nodes_len;
-      // NB: there are more fields in this struct, but we can safely (?)
-      // ignore them because we never have collections of this type.
-    } RawNodes;
-
-    void externs_set(ExternContext*,
-                     extern_log,
-                     extern_key_for,
-                     extern_val_for,
-                     extern_clone_val,
-                     extern_drop_handles,
-                     extern_id_to_str,
-                     extern_val_to_str,
-                     extern_satisfied_by,
-                     extern_satisfied_by_type,
-                     extern_store_list,
-                     extern_store_bytes,
-                     extern_project,
-                     extern_project_ignoring_type,
-                     extern_project_multi,
-                     extern_create_exception,
-                     extern_invoke_runnable,
-                     TypeId);
-
-    RawScheduler* scheduler_create(Function,
-                                   Function,
-                                   Function,
-                                   Function,
-                                   Function,
-                                   Function,
-                                   Function,
-                                   Function,
-                                   TypeConstraint,
-                                   TypeConstraint,
-                                   TypeConstraint,
-                                   TypeConstraint,
-                                   TypeConstraint,
-                                   TypeConstraint,
-                                   TypeConstraint,
-                                   TypeConstraint,
-                                   TypeConstraint,
-                                   TypeConstraint,
-                                   TypeId,
-                                   TypeId,
-                                   Buffer,
-                                   BufferBuffer);
-    void scheduler_post_fork(RawScheduler*);
-    void scheduler_destroy(RawScheduler*);
-
-    void intrinsic_task_add(RawScheduler*, Function, TypeId, TypeConstraint, TypeConstraint);
-    void singleton_task_add(RawScheduler*, Function, TypeConstraint);
-
-    void task_add(RawScheduler*, Function, TypeConstraint);
-    void task_add_select(RawScheduler*, TypeConstraint);
-    void task_add_select_variant(RawScheduler*, TypeConstraint, Buffer);
-    void task_add_select_literal(RawScheduler*, Key, TypeConstraint);
-    void task_add_select_dependencies(RawScheduler*, TypeConstraint, TypeConstraint, Buffer, TypeIdBuffer, bool);
-    void task_add_select_projection(RawScheduler*, TypeConstraint, TypeConstraint, Buffer, TypeConstraint);
-    void task_end(RawScheduler*);
-
-    uint64_t graph_len(RawScheduler*);
-    uint64_t graph_invalidate(RawScheduler*, BufferBuffer);
-    void graph_visualize(RawScheduler*, char*);
-    void graph_trace(RawScheduler*, char*);
-
-
-    void execution_reset(RawScheduler*);
-    void execution_add_root_select(RawScheduler*, Key, TypeConstraint);
-    void execution_add_root_select_dependencies(RawScheduler*,
-                                                Key,
-                                                TypeConstraint,
-                                                TypeConstraint,
-                                                Buffer,
-                                                TypeIdBuffer,
-                                                bool);
-    ExecutionStat execution_execute(RawScheduler*);
-    RawNodes* execution_roots(RawScheduler*);
-
-    Value validator_run(RawScheduler*, TypeId*, uint64_t);
-
-    void rule_graph_visualize(RawScheduler*, TypeId*, uint64_t, char*);
-    void rule_subgraph_visualize(RawScheduler*, TypeId, TypeConstraint, char*);
-
-    void nodes_destroy(RawNodes*);
-    '''
+NATIVE_ENGINE_MODULE = 'native_engine'
+
+
+CFFI_TYPEDEFS = '''
+typedef uint64_t   Id;
+typedef void*      Handle;
+
+typedef struct {
+  Id id_;
+} TypeId;
+
+typedef struct {
+  Id id_;
+} TypeConstraint;
+
+typedef struct {
+  Id id_;
+} Function;
+
+typedef struct {
+  Handle   handle;
+} Value;
+
+typedef struct {
+  Id       id_;
+  TypeId   type_id;
+} Key;
+
+typedef struct {
+  uint8_t*  bytes_ptr;
+  uint64_t  bytes_len;
+  Value     handle_;
+} Buffer;
+
+typedef struct {
+  Value*     values_ptr;
+  uint64_t   values_len;
+  Value      handle_;
+} ValueBuffer;
+
+typedef struct {
+  TypeId*     ids_ptr;
+  uint64_t    ids_len;
+  Value       handle_;
+} TypeIdBuffer;
+
+typedef struct {
+  Buffer*     bufs_ptr;
+  uint64_t    bufs_len;
+  Value       handle_;
+} BufferBuffer;
+
+typedef struct {
+  Value  value;
+  _Bool  is_throw;
+} RunnableComplete;
+
+typedef uint64_t EntryId;
+
+typedef void ExternContext;
+
+// On the rust side the integration is defined in externs.rs
+typedef void             (*extern_ptr_log)(ExternContext*, uint8_t, uint8_t*, uint64_t);
+typedef Key              (*extern_ptr_key_for)(ExternContext*, Value*);
+typedef Value            (*extern_ptr_val_for)(ExternContext*, Key*);
+typedef Value            (*extern_ptr_clone_val)(ExternContext*, Value*);
+typedef void             (*extern_ptr_drop_handles)(ExternContext*, Handle*, uint64_t);
+typedef Buffer           (*extern_ptr_id_to_str)(ExternContext*, Id);
+typedef Buffer           (*extern_ptr_val_to_str)(ExternContext*, Value*);
+typedef _Bool            (*extern_ptr_satisfied_by)(ExternContext*, TypeConstraint*, Value*);
+typedef _Bool            (*extern_ptr_satisfied_by_type)(ExternContext*, TypeConstraint*, TypeId*);
+typedef Value            (*extern_ptr_store_list)(ExternContext*, Value**, uint64_t, _Bool);
+typedef Value            (*extern_ptr_store_bytes)(ExternContext*, uint8_t*, uint64_t);
+typedef Value            (*extern_ptr_project)(ExternContext*, Value*, uint8_t*, uint64_t, TypeId*);
+typedef ValueBuffer      (*extern_ptr_project_multi)(ExternContext*, Value*, uint8_t*, uint64_t);
+typedef Value            (*extern_ptr_project_ignoring_type)(ExternContext*, Value*, uint8_t*, uint64_t);
+typedef Value            (*extern_ptr_create_exception)(ExternContext*, uint8_t*, uint64_t);
+typedef RunnableComplete (*extern_ptr_invoke_runnable)(ExternContext*, Value*, Value*, uint64_t, _Bool);
+
+typedef void RawScheduler;
+
+typedef struct {
+  uint64_t runnable_count;
+  uint64_t scheduling_iterations;
+} ExecutionStat;
+
+typedef struct {
+  Key             subject;
+  TypeConstraint  product;
+  uint8_t         state_tag;
+  Value           state_value;
+} RawNode;
+
+typedef struct {
+  RawNode*  nodes_ptr;
+  uint64_t  nodes_len;
+  // NB: there are more fields in this struct, but we can safely (?)
+  // ignore them because we never have collections of this type.
+} RawNodes;
+'''
+
+CFFI_HEADERS = '''
+void externs_set(ExternContext*,
+                 extern_ptr_log,
+                 extern_ptr_key_for,
+                 extern_ptr_val_for,
+                 extern_ptr_clone_val,
+                 extern_ptr_drop_handles,
+                 extern_ptr_id_to_str,
+                 extern_ptr_val_to_str,
+                 extern_ptr_satisfied_by,
+                 extern_ptr_satisfied_by_type,
+                 extern_ptr_store_list,
+                 extern_ptr_store_bytes,
+                 extern_ptr_project,
+                 extern_ptr_project_ignoring_type,
+                 extern_ptr_project_multi,
+                 extern_ptr_create_exception,
+                 extern_ptr_invoke_runnable,
+                 TypeId);
+
+RawScheduler* scheduler_create(Function,
+                               Function,
+                               Function,
+                               Function,
+                               Function,
+                               Function,
+                               Function,
+                               Function,
+                               TypeConstraint,
+                               TypeConstraint,
+                               TypeConstraint,
+                               TypeConstraint,
+                               TypeConstraint,
+                               TypeConstraint,
+                               TypeConstraint,
+                               TypeConstraint,
+                               TypeConstraint,
+                               TypeConstraint,
+                               TypeId,
+                               TypeId,
+                               Buffer,
+                               BufferBuffer);
+void scheduler_post_fork(RawScheduler*);
+void scheduler_destroy(RawScheduler*);
+
+void intrinsic_task_add(RawScheduler*, Function, TypeId, TypeConstraint, TypeConstraint);
+void singleton_task_add(RawScheduler*, Function, TypeConstraint);
+
+void task_add(RawScheduler*, Function, TypeConstraint);
+void task_add_select(RawScheduler*, TypeConstraint);
+void task_add_select_variant(RawScheduler*, TypeConstraint, Buffer);
+void task_add_select_literal(RawScheduler*, Key, TypeConstraint);
+void task_add_select_dependencies(RawScheduler*, TypeConstraint, TypeConstraint, Buffer, TypeIdBuffer, _Bool);
+void task_add_select_projection(RawScheduler*, TypeConstraint, TypeConstraint, Buffer, TypeConstraint);
+void task_end(RawScheduler*);
+
+uint64_t graph_len(RawScheduler*);
+uint64_t graph_invalidate(RawScheduler*, BufferBuffer);
+void graph_visualize(RawScheduler*, char*);
+void graph_trace(RawScheduler*, char*);
+
+void execution_reset(RawScheduler*);
+void execution_add_root_select(RawScheduler*, Key, TypeConstraint);
+void execution_add_root_select_dependencies(RawScheduler*,
+                                            Key,
+                                            TypeConstraint,
+                                            TypeConstraint,
+                                            Buffer,
+                                            TypeIdBuffer,
+                                            _Bool);
+ExecutionStat execution_execute(RawScheduler*);
+RawNodes* execution_roots(RawScheduler*);
+
+Value validator_run(RawScheduler*, TypeId*, uint64_t);
+
+void rule_graph_visualize(RawScheduler*, TypeId*, uint64_t, char*);
+void rule_subgraph_visualize(RawScheduler*, TypeId, TypeConstraint, char*);
+
+void nodes_destroy(RawNodes*);
+'''
+
+CFFI_EXTERNS = '''
+extern "Python" {
+  void             extern_log(ExternContext*, uint8_t, uint8_t*, uint64_t);
+  Key              extern_key_for(ExternContext*, Value*);
+  Value            extern_val_for(ExternContext*, Key*);
+  Value            extern_clone_val(ExternContext*, Value*);
+  void             extern_drop_handles(ExternContext*, Handle*, uint64_t);
+  Buffer           extern_id_to_str(ExternContext*, Id);
+  Buffer           extern_val_to_str(ExternContext*, Value*);
+  _Bool            extern_satisfied_by(ExternContext*, TypeConstraint*, Value*);
+  _Bool            extern_satisfied_by_type(ExternContext*, TypeConstraint*, TypeId*);
+  Value            extern_store_list(ExternContext*, Value**, uint64_t, _Bool);
+  Value            extern_store_bytes(ExternContext*, uint8_t*, uint64_t);
+  Value            extern_project(ExternContext*, Value*, uint8_t*, uint64_t, TypeId*);
+  Value            extern_project_ignoring_type(ExternContext*, Value*, uint8_t*, uint64_t);
+  ValueBuffer      extern_project_multi(ExternContext*, Value*, uint8_t*, uint64_t);
+  Value            extern_create_exception(ExternContext*, uint8_t*, uint64_t);
+  RunnableComplete extern_invoke_runnable(ExternContext*, Value*, Value*, uint64_t, _Bool);
+}
+'''
+
+
+def get_build_cflags():
+  """Synthesize a CFLAGS env var from the current python env for building of C modules."""
+  return '{} {} -I{}'.format(
+    sysconfig.get_config_var('BASECFLAGS'),
+    sysconfig.get_config_var('OPT'),
+    sysconfig.get_path('include')
   )
 
 
-@_FFI.callback("void(ExternContext*, uint8_t, uint8_t*, uint64_t)")
-def extern_log(context_handle, level, msg_ptr, msg_len):
-  """Given a log level and utf8 message string, log it."""
-  msg = bytes(_FFI.buffer(msg_ptr, msg_len)).decode('utf-8')
-  if level == 0:
-    logger.debug(msg)
-  elif level == 1:
-    logger.info(msg)
-  elif level == 2:
-    logger.warn(msg)
-  else:
-    logger.critical(msg)
-
-
-@_FFI.callback("Key(ExternContext*, Value*)")
-def extern_key_for(context_handle, val):
-  """Return a Key for a Value."""
-  c = _FFI.from_handle(context_handle)
-  return c.to_key(c.from_value(val))
-
-
-@_FFI.callback("Value(ExternContext*, Key*)")
-def extern_val_for(context_handle, key):
-  """Return a Value for a Key."""
-  c = _FFI.from_handle(context_handle)
-  return c.to_value(c.from_key(key))
-
-
-@_FFI.callback("Value(ExternContext*, Value*)")
-def extern_clone_val(context_handle, val):
-  """Clone the given Value."""
-  c = _FFI.from_handle(context_handle)
-  return c.to_value(c.from_value(val))
-
-
-@_FFI.callback("void(ExternContext*, Handle*, uint64_t)")
-def extern_drop_handles(context_handle, handles_ptr, handles_len):
-  """Drop the given Handles."""
-  c = _FFI.from_handle(context_handle)
-  handles = _FFI.unpack(handles_ptr, handles_len)
-  c.drop_handles(handles)
-
-
-@_FFI.callback("Buffer(ExternContext*, Id)")
-def extern_id_to_str(context_handle, id_):
-  """Given an Id for `obj`, write str(obj) and return it."""
-  c = _FFI.from_handle(context_handle)
-  return c.utf8_buf(six.text_type(c.from_id(id_)))
-
-
-@_FFI.callback("Buffer(ExternContext*, Value*)")
-def extern_val_to_str(context_handle, val):
-  """Given a Value for `obj`, write str(obj) and return it."""
-  c = _FFI.from_handle(context_handle)
-  return c.utf8_buf(six.text_type(c.from_value(val)))
-
-
-@_FFI.callback("bool(ExternContext*, TypeConstraint*, Value*)")
-def extern_satisfied_by(context_handle, constraint_id, val):
-  """Given a TypeConstraint and a Value return constraint.satisfied_by(value)."""
-  c = _FFI.from_handle(context_handle)
-  return c.from_id(constraint_id.id_).satisfied_by(c.from_value(val))
-
-
-@_FFI.callback("bool(ExternContext*, TypeConstraint*, TypeId*)")
-def extern_satisfied_by_type(context_handle, constraint_id, cls_id):
-  """Given a TypeConstraint and a TypeId, return constraint.satisfied_by_type(type_id)."""
-  c = _FFI.from_handle(context_handle)
-  return c.from_id(constraint_id.id_).satisfied_by_type(c.from_id(cls_id.id_))
-
-
-@_FFI.callback("Value(ExternContext*, Value**, uint64_t, bool)")
-def extern_store_list(context_handle, vals_ptr_ptr, vals_len, merge):
-  """Given storage and an array of Values, return a new Value to represent the list."""
-  c = _FFI.from_handle(context_handle)
-  vals = tuple(c.from_value(val) for val in _FFI.unpack(vals_ptr_ptr, vals_len))
-  if merge:
-    # Expect each obj to represent a list, and do a de-duping merge.
-    merged_set = set()
-    def merged():
-      for outer_val in vals:
-        for inner_val in outer_val:
-          if inner_val in merged_set:
-            continue
-          merged_set.add(inner_val)
-          yield inner_val
-    vals = tuple(merged())
-  return c.to_value(vals)
-
-
-@_FFI.callback("Value(ExternContext*, uint8_t*, uint64_t)")
-def extern_store_bytes(context_handle, bytes_ptr, bytes_len):
-  """Given a context and raw bytes, return a new Value to represent the content."""
-  c = _FFI.from_handle(context_handle)
-  return c.to_value(bytes(_FFI.buffer(bytes_ptr, bytes_len)))
-
-
-@_FFI.callback("Value(ExternContext*, Value*, uint8_t*, uint64_t, TypeId*)")
-def extern_project(context_handle, val, field_str_ptr, field_str_len, type_id):
-  """Given a Value for `obj`, a field name, and a type, project the field as a new Value."""
-  c = _FFI.from_handle(context_handle)
-  obj = c.from_value(val)
-  field_name = to_py_str(field_str_ptr, field_str_len)
-  typ = c.from_id(type_id.id_)
-
-  projected = getattr(obj, field_name)
-  if type(projected) is not typ:
-    projected = typ(projected)
-
-  return c.to_value(projected)
-
-
-@_FFI.callback("Value(ExternContext*, Value*, uint8_t*, uint64_t)")
-def extern_project_ignoring_type(context_handle, val, field_str_ptr, field_str_len):
-  """Given a Value for `obj`, and a field name, project the field as a new Value."""
-  c = _FFI.from_handle(context_handle)
-  obj = c.from_value(val)
-  field_name = to_py_str(field_str_ptr, field_str_len)
-  projected = getattr(obj, field_name)
-
-  return c.to_value(projected)
-
-
-@_FFI.callback("ValueBuffer(ExternContext*, Value*, uint8_t*, uint64_t)")
-def extern_project_multi(context_handle, val, field_str_ptr, field_str_len):
-  """Given a Key for `obj`, and a field name, project the field as a list of Keys."""
-  c = _FFI.from_handle(context_handle)
-  obj = c.from_value(val)
-  field_name = to_py_str(field_str_ptr, field_str_len)
-
-  return c.vals_buf(tuple(c.to_value(p) for p in getattr(obj, field_name)))
-
-
-@_FFI.callback("Value(ExternContext*, uint8_t*, uint64_t)")
-def extern_create_exception(context_handle, msg_ptr, msg_len):
-  """Given a utf8 message string, create an Exception object."""
-  c = _FFI.from_handle(context_handle)
-  msg = to_py_str(msg_ptr, msg_len)
-  return c.to_value(Exception(msg))
-
-
-def to_py_str(msg_ptr, msg_len):
-  return bytes(_FFI.buffer(msg_ptr, msg_len)).decode('utf-8')
-
-
-@_FFI.callback("RunnableComplete(ExternContext*, Value*, Value*, uint64_t, bool)")
-def extern_invoke_runnable(context_handle, func, args_ptr, args_len, cacheable):
-  """Given a destructured rawRunnable, run it."""
-  c = _FFI.from_handle(context_handle)
-  runnable = c.from_value(func)
-  args = tuple(c.from_value(arg) for arg in _FFI.unpack(args_ptr, args_len))
-
-  try:
-    val = runnable(*args)
-    is_throw = False
-  except Exception as e:
-    val = e
-    is_throw = True
-
-  return RunnableComplete(c.to_value(val), is_throw)
+def bootstrap_c_source(output_dir, module_name=NATIVE_ENGINE_MODULE):
+  """Bootstrap an external CFFI C source file."""
+  safe_mkdir(output_dir)
+
+  output_prefix = os.path.join(output_dir, module_name)
+  c_file = '{}.c'.format(output_prefix)
+  env_script = '{}.sh'.format(output_prefix)
+
+  ffibuilder = cffi.FFI()
+  ffibuilder.cdef(CFFI_TYPEDEFS)
+  ffibuilder.cdef(CFFI_HEADERS)
+  ffibuilder.cdef(CFFI_EXTERNS)
+  ffibuilder.set_source(module_name, CFFI_TYPEDEFS + CFFI_HEADERS)
+  ffibuilder.emit_c_code(six.binary_type(c_file))
+
+  # Write a shell script to be sourced at build time that contains inherited CFLAGS.
+  print('generating {}'.format(env_script))
+  with open(env_script, 'wb') as f:
+    f.write('export CFLAGS="{}"\n'.format(get_build_cflags()))
+
+
+def _initialize_externs(ffi):
+  """Initializes extern callbacks given a CFFI handle."""
+
+  def to_py_str(msg_ptr, msg_len):
+    return bytes(ffi.buffer(msg_ptr, msg_len)).decode('utf-8')
+
+  @ffi.def_extern()
+  def extern_log(context_handle, level, msg_ptr, msg_len):
+    """Given a log level and utf8 message string, log it."""
+    msg = bytes(ffi.buffer(msg_ptr, msg_len)).decode('utf-8')
+    if level == 0:
+      logger.debug(msg)
+    elif level == 1:
+      logger.info(msg)
+    elif level == 2:
+      logger.warn(msg)
+    else:
+      logger.critical(msg)
+
+  @ffi.def_extern()
+  def extern_key_for(context_handle, val):
+    """Return a Key for a Value."""
+    c = ffi.from_handle(context_handle)
+    return c.to_key(c.from_value(val))
+
+  @ffi.def_extern()
+  def extern_val_for(context_handle, key):
+    """Return a Value for a Key."""
+    c = ffi.from_handle(context_handle)
+    return c.to_value(c.from_key(key))
+
+  @ffi.def_extern()
+  def extern_clone_val(context_handle, val):
+    """Clone the given Value."""
+    c = ffi.from_handle(context_handle)
+    return c.to_value(c.from_value(val))
+
+  @ffi.def_extern()
+  def extern_drop_handles(context_handle, handles_ptr, handles_len):
+    """Drop the given Handles."""
+    c = ffi.from_handle(context_handle)
+    handles = ffi.unpack(handles_ptr, handles_len)
+    c.drop_handles(handles)
+
+  @ffi.def_extern()
+  def extern_id_to_str(context_handle, id_):
+    """Given an Id for `obj`, write str(obj) and return it."""
+    c = ffi.from_handle(context_handle)
+    return c.utf8_buf(six.text_type(c.from_id(id_)))
+
+  @ffi.def_extern()
+  def extern_val_to_str(context_handle, val):
+    """Given a Value for `obj`, write str(obj) and return it."""
+    c = ffi.from_handle(context_handle)
+    return c.utf8_buf(six.text_type(c.from_value(val)))
+
+  @ffi.def_extern()
+  def extern_satisfied_by(context_handle, constraint_id, val):
+    """Given a TypeConstraint and a Value return constraint.satisfied_by(value)."""
+    c = ffi.from_handle(context_handle)
+    return c.from_id(constraint_id.id_).satisfied_by(c.from_value(val))
+
+  @ffi.def_extern()
+  def extern_satisfied_by_type(context_handle, constraint_id, cls_id):
+    """Given a TypeConstraint and a TypeId, return constraint.satisfied_by_type(type_id)."""
+    c = ffi.from_handle(context_handle)
+    return c.from_id(constraint_id.id_).satisfied_by_type(c.from_id(cls_id.id_))
+
+  @ffi.def_extern()
+  def extern_store_list(context_handle, vals_ptr_ptr, vals_len, merge):
+    """Given storage and an array of Values, return a new Value to represent the list."""
+    c = ffi.from_handle(context_handle)
+    vals = tuple(c.from_value(val) for val in ffi.unpack(vals_ptr_ptr, vals_len))
+    if merge:
+      # Expect each obj to represent a list, and do a de-duping merge.
+      merged_set = set()
+      def merged():
+        for outer_val in vals:
+          for inner_val in outer_val:
+            if inner_val in merged_set:
+              continue
+            merged_set.add(inner_val)
+            yield inner_val
+      vals = tuple(merged())
+    return c.to_value(vals)
+
+  @ffi.def_extern()
+  def extern_store_bytes(context_handle, bytes_ptr, bytes_len):
+    """Given a context and raw bytes, return a new Value to represent the content."""
+    c = ffi.from_handle(context_handle)
+    return c.to_value(bytes(ffi.buffer(bytes_ptr, bytes_len)))
+
+  @ffi.def_extern()
+  def extern_project(context_handle, val, field_str_ptr, field_str_len, type_id):
+    """Given a Value for `obj`, a field name, and a type, project the field as a new Value."""
+    c = ffi.from_handle(context_handle)
+    obj = c.from_value(val)
+    field_name = to_py_str(field_str_ptr, field_str_len)
+    typ = c.from_id(type_id.id_)
+
+    projected = getattr(obj, field_name)
+    if type(projected) is not typ:
+      projected = typ(projected)
+
+    return c.to_value(projected)
+
+  @ffi.def_extern()
+  def extern_project_ignoring_type(context_handle, val, field_str_ptr, field_str_len):
+    """Given a Value for `obj`, and a field name, project the field as a new Value."""
+    c = ffi.from_handle(context_handle)
+    obj = c.from_value(val)
+    field_name = to_py_str(field_str_ptr, field_str_len)
+    projected = getattr(obj, field_name)
+
+    return c.to_value(projected)
+
+  @ffi.def_extern()
+  def extern_project_multi(context_handle, val, field_str_ptr, field_str_len):
+    """Given a Key for `obj`, and a field name, project the field as a list of Keys."""
+    c = ffi.from_handle(context_handle)
+    obj = c.from_value(val)
+    field_name = to_py_str(field_str_ptr, field_str_len)
+
+    return c.vals_buf(tuple(c.to_value(p) for p in getattr(obj, field_name)))
+
+  @ffi.def_extern()
+  def extern_create_exception(context_handle, msg_ptr, msg_len):
+    """Given a utf8 message string, create an Exception object."""
+    c = ffi.from_handle(context_handle)
+    msg = to_py_str(msg_ptr, msg_len)
+    return c.to_value(Exception(msg))
+
+  @ffi.def_extern()
+  def extern_invoke_runnable(context_handle, func, args_ptr, args_len, cacheable):
+    """Given a destructured rawRunnable, run it."""
+    c = ffi.from_handle(context_handle)
+    runnable = c.from_value(func)
+    args = tuple(c.from_value(arg) for arg in ffi.unpack(args_ptr, args_len))
+
+    try:
+      val = runnable(*args)
+      is_throw = False
+    except Exception as e:
+      val = e
+      is_throw = True
+
+    return RunnableComplete(c.to_value(val), is_throw)
 
 
 class Value(datatype('Value', ['handle'])):
@@ -431,10 +475,15 @@ def get(self, oid):
 class ExternContext(object):
   """A wrapper around python objects used in static extern functions in this module."""
 
-  def __init__(self):
+  def __init__(self, ffi):
+    """
+    :param CompiledCFFI ffi: The CFFI handle to the compiled native engine lib.
+    """
+    self._ffi = ffi
+
     # A handle to this object to ensure that the native wrapper survives at least as
     # long as this object.
-    self.handle = _FFI.new_handle(self)
+    self.handle = self._ffi.new_handle(self)
 
     # The native code will invoke externs concurrently, so locking is needed around
     # datastructures in this context.
@@ -450,7 +499,7 @@ def __init__(self):
     self._handles = set()
 
   def buf(self, bytestring):
-    buf = _FFI.new('uint8_t[]', bytestring)
+    buf = self._ffi.new('uint8_t[]', bytestring)
     return (buf, len(bytestring), self.to_value(buf))
 
   def utf8_buf(self, string):
@@ -458,24 +507,24 @@ def utf8_buf(self, string):
 
   def utf8_buf_buf(self, strings):
     bufs = [self.utf8_buf(string) for string in strings]
-    buf_buf = _FFI.new('Buffer[]', bufs)
+    buf_buf = self._ffi.new('Buffer[]', bufs)
     return (buf_buf, len(bufs), self.to_value(buf_buf))
 
   def vals_buf(self, keys):
-    buf = _FFI.new('Value[]', keys)
+    buf = self._ffi.new('Value[]', keys)
     return (buf, len(keys), self.to_value(buf))
 
   def type_ids_buf(self, types):
-    buf = _FFI.new('TypeId[]', types)
+    buf = self._ffi.new('TypeId[]', types)
     return (buf, len(types), self.to_value(buf))
 
   def to_value(self, obj):
-    handle = _FFI.new_handle(obj)
+    handle = self._ffi.new_handle(obj)
     self._handles.add(handle)
     return Value(handle)
 
   def from_value(self, val):
-    return _FFI.from_handle(val.handle)
+    return self._ffi.from_handle(val.handle)
 
   def drop_handles(self, handles):
     self._handles -= set(handles)
@@ -549,58 +598,82 @@ def __init__(self, binary_util, version, supportdir, visualize_to_dir):
   def visualize_to_dir(self):
     return self._visualize_to_dir
 
+  @memoized_property
+  def binary(self):
+    """Load and return the path to the native engine binary."""
+    return self._binary_util.select_binary(self._supportdir,
+                                           self._version,
+                                           '{}.so'.format(NATIVE_ENGINE_MODULE))
+
   @memoized_property
   def lib(self):
-    """Load and return the `libgraph` module."""
-    binary = self._binary_util.select_binary(self._supportdir,
-                                              self._version,
-                                              'native-engine')
-    return _FFI.dlopen(binary)
+    """Load and return the native engine module."""
+    return self.ffi.dlopen(self.binary)
+
+  @memoized_property
+  def ffi(self):
+    """A CompiledCFFI handle as imported from the native engine python module."""
+    ffi = getattr(self._ffi_module, 'ffi')
+    _initialize_externs(ffi)
+    return ffi
+
+  @memoized_property
+  def ffi_lib(self):
+    """A CFFI Library handle as imported from the native engine python module."""
+    return getattr(self._ffi_module, 'lib')
+
+  @memoized_property
+  def _ffi_module(self):
+    """Load the native engine as a python module and register CFFI externs."""
+    native_bin_dir = os.path.dirname(self.binary)
+    logger.debug('loading native engine python module from: %s', native_bin_dir)
+    sys.path.insert(0, native_bin_dir)
+    return importlib.import_module(NATIVE_ENGINE_MODULE)
 
   @memoized_property
   def context(self):
     # We statically initialize a ExternContext to correspond to the queue of dropped
     # Handles that the native code maintains.
     def init_externs():
-      context = ExternContext()
+      context = ExternContext(self.ffi)
       self.lib.externs_set(context.handle,
-                           extern_log,
-                           extern_key_for,
-                           extern_val_for,
-                           extern_clone_val,
-                           extern_drop_handles,
-                           extern_id_to_str,
-                           extern_val_to_str,
-                           extern_satisfied_by,
-                           extern_satisfied_by_type,
-                           extern_store_list,
-                           extern_store_bytes,
-                           extern_project,
-                           extern_project_ignoring_type,
-                           extern_project_multi,
-                           extern_create_exception,
-                           extern_invoke_runnable,
+                           self.ffi_lib.extern_log,
+                           self.ffi_lib.extern_key_for,
+                           self.ffi_lib.extern_val_for,
+                           self.ffi_lib.extern_clone_val,
+                           self.ffi_lib.extern_drop_handles,
+                           self.ffi_lib.extern_id_to_str,
+                           self.ffi_lib.extern_val_to_str,
+                           self.ffi_lib.extern_satisfied_by,
+                           self.ffi_lib.extern_satisfied_by_type,
+                           self.ffi_lib.extern_store_list,
+                           self.ffi_lib.extern_store_bytes,
+                           self.ffi_lib.extern_project,
+                           self.ffi_lib.extern_project_ignoring_type,
+                           self.ffi_lib.extern_project_multi,
+                           self.ffi_lib.extern_create_exception,
+                           self.ffi_lib.extern_invoke_runnable,
                            TypeId(context.to_id(str)))
       return context
 
-    return _FFI.init_once(init_externs, 'ExternContext singleton')
+    return self.ffi.init_once(init_externs, 'ExternContext singleton')
 
   def new(self, cdecl, init):
-    return _FFI.new(cdecl, init)
+    return self.ffi.new(cdecl, init)
 
   def gc(self, cdata, destructor):
     """Register a method to be called when `cdata` is garbage collected.
 
     Returns a new reference that should be used in place of `cdata`.
     """
-    return _FFI.gc(cdata, destructor)
+    return self.ffi.gc(cdata, destructor)
 
   def unpack(self, cdata_ptr, count):
     """Given a pointer representing an array, and its count of entries, return a list."""
-    return _FFI.unpack(cdata_ptr, count)
+    return self.ffi.unpack(cdata_ptr, count)
 
   def buffer(self, cdata):
-    return _FFI.buffer(cdata)
+    return self.ffi.buffer(cdata)
 
   def new_scheduler(self,
                     build_root,


Test patch: