Repository: pantsbuild/pants

Issue:
New python pipeline regresses on setup-py target owner checks for codegen.
Aurora sees this when trying to upgrade to `1.4.0.dev22`:
```
$ ./pants setup-py --recursive src/main/python/apache/aurora/admin
    
    00:17:40 00:00 [main]
                   (To run a reporting server: ./pants server)
    00:17:40 00:00   [setup]
    00:17:40 00:00     [parse]
                   Executing tasks in goals: gen -> pyprep -> setup-py
    00:17:40 00:00   [gen]
    00:17:40 00:00     [thrift-py]
    00:17:40 00:00   [pyprep]
    00:17:40 00:00     [interpreter]
    00:17:40 00:00     [requirements]
    00:17:40 00:00     [sources]
    00:17:40 00:00   [setup-py]
    00:17:40 00:00     [setup-py]
                       Creating setup.py project for PythonLibrary(BuildFileAddress(src/main/python/apache/aurora/admin/BUILD, admin))
                       Creating setup.py project for PythonLibrary(BuildFileAddress(api/src/main/thrift/org/apache/aurora/gen/BUILD, gen))
    FAILURE: No exported target owner found for PythonLibrary(.pants.d/gen/thrift-py/252d64521cf9/api.src.main.thrift.org.apache.aurora.gen._api/current:api.src.main.thrift.org.apache.aurora.gen._api)
    
    00:17:40 00:00   [complete]
                   FAILURE
```

Previously, this worked.


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/codegen/thrift/python/apache_thrift_py_gen.py b/src/python/pants/backend/codegen/thrift/python/apache_thrift_py_gen.py
--- a/src/python/pants/backend/codegen/thrift/python/apache_thrift_py_gen.py
+++ b/src/python/pants/backend/codegen/thrift/python/apache_thrift_py_gen.py
@@ -10,7 +10,7 @@
 from pants.backend.codegen.thrift.lib.apache_thrift_gen_base import ApacheThriftGenBase
 from pants.backend.codegen.thrift.python.python_thrift_library import PythonThriftLibrary
 from pants.backend.python.targets.python_library import PythonLibrary
-from pants.util.dirutil import safe_delete
+from pants.util.dirutil import safe_delete, safe_walk
 
 
 class ApacheThriftPyGen(ApacheThriftGenBase):
@@ -26,9 +26,28 @@ def synthetic_target_type(self, target):
 
   def execute_codegen(self, target, target_workdir):
     super(ApacheThriftPyGen, self).execute_codegen(target, target_workdir)
-    # Thrift puts an __init__.py file at the root, and we don't want one there
-    # (it's not needed, and it confuses some import mechanisms).
-    safe_delete(os.path.join(target_workdir, '__init__.py'))
+
+    # Thrift generates code with all parent namespaces with empty __init__.py's. Since pants allows
+    # splitting a thrift namespace hierarchy across multiple packages, we explicitly insert
+    # namespace packages to allow for consumption of 2 or more of these packages in the same
+    # PYTHONPATH.
+    for root, _, files in safe_walk(target_workdir):
+      if '__init__.py' not in files:  # skip non-packages
+        continue
+
+      init_py_abspath = os.path.join(root, '__init__.py')
+
+      # Thrift puts an __init__.py file at the root, and we don't want one there (it's not needed,
+      # and it confuses some import mechanisms).
+      if root == target_workdir:
+        safe_delete(init_py_abspath)
+      elif os.path.getsize(init_py_abspath) == 0:  # empty __init__, translate to namespace package
+        with open(init_py_abspath, 'wb') as f:
+          f.write(b"__import__('pkg_resources').declare_namespace(__name__)")
+      else:
+        # A non-empty __init__, this is a leaf package, usually with ttypes and constants; so we
+        # leave as-is.
+        pass
 
   def ignore_dup(self, tgt1, tgt2, rel_src):
     # Thrift generates all the intermediate __init__.py files, and they shouldn't
diff --git a/src/python/pants/backend/python/tasks2/setup_py.py b/src/python/pants/backend/python/tasks2/setup_py.py
--- a/src/python/pants/backend/python/tasks2/setup_py.py
+++ b/src/python/pants/backend/python/tasks2/setup_py.py
@@ -31,6 +31,7 @@
 from pants.build_graph.resources import Resources
 from pants.task.task import Task
 from pants.util.dirutil import safe_rmtree, safe_walk
+from pants.util.memo import memoized_property
 from pants.util.meta import AbstractClass
 
 
@@ -231,7 +232,7 @@ def reduced_dependencies(self, exported_target):
     #     3rdparty since these may be introduced by exported subgraphs we discover in later steps!
     # 2.) Determine the owner of each target collected in 1 by walking the ancestor chain to find
     #     the closest exported target.  The ancestor chain is just all targets whose spec path is
-    #     a prefix of th descendant.  In other words, all targets in descendant's BUILD file family
+    #     a prefix of the descendant.  In other words, all targets in descendant's BUILD file family
     #     (its siblings), all targets in its parent directory BUILD file family, and so on.
     # 3.) Finally walk the exported target once more, replacing each visited dependency with its
     #     owner.
@@ -242,11 +243,15 @@ def reduced_dependencies(self, exported_target):
 
     owner_by_owned_python_target = OrderedDict()
 
+    # Only check ownership on the original target graph.
+    original_exported_target = exported_target.derived_from
+
     def collect_potentially_owned_python_targets(current):
-      owner_by_owned_python_target[current] = None  # We can't know the owner in the 1st pass.
+      if current.is_original:
+        owner_by_owned_python_target[current] = None  # We can't know the owner in the 1st pass.
       return (current == exported_target) or not self.is_exported(current)
 
-    self._walk(exported_target, collect_potentially_owned_python_targets)
+    self._walk(original_exported_target, collect_potentially_owned_python_targets)
 
     for owned in owner_by_owned_python_target:
       if self.requires_export(owned) and not self.is_exported(owned):
@@ -288,7 +293,7 @@ def collect_reduced_dependencies(current):
         return owner == exported_target or not self.requires_export(current)
 
     self._walk(exported_target, collect_reduced_dependencies)
-    return reduced_dependencies
+    return OrderedSet(d for d in reduced_dependencies if d.is_original)
 
 
 class SetupPy(Task):
@@ -458,6 +463,11 @@ def __init__(self, *args, **kwargs):
     self._run = self.get_options().run
     self._recursive = self.get_options().recursive
 
+  @memoized_property
+  def derived_by_original(self):
+    derived = self.context.targets(predicate=lambda t: not t.is_original)
+    return {t.derived_from: t for t in derived}
+
   def write_contents(self, root_target, reduced_dependencies, chroot):
     """Write contents of the target."""
     def write_target_source(target, src):
@@ -475,11 +485,14 @@ def write_target_source(target, src):
                       os.path.join(self.SOURCE_ROOT, src, '__init__.py'))
 
     def write_target(target):
-      for rel_source in target.sources_relative_to_buildroot():
+      # We want to operate on the final sources target owns, so we potentially replace it with
+      # the target derived from it (by a codegen task).
+      subject = self.derived_by_original.get(target, target)
+      for rel_source in subject.sources_relative_to_buildroot():
         abs_source_path = os.path.join(get_buildroot(), rel_source)
-        abs_source_root_path = os.path.join(get_buildroot(), target.target_base)
+        abs_source_root_path = os.path.join(get_buildroot(), subject.target_base)
         source_root_relative_path = os.path.relpath(abs_source_path, abs_source_root_path)
-        write_target_source(target, source_root_relative_path)
+        write_target_source(subject, source_root_relative_path)
 
     write_target(root_target)
     for dependency in reduced_dependencies:
@@ -576,24 +589,14 @@ def create_setup_py(self, target, dist_dir):
     return setup_dir, reduced_deps
 
   def execute(self):
-    # We operate on the target roots, except that we replace codegen targets with their
-    # corresponding synthetic targets, since those have the generated sources that actually
-    # get published. Note that the "provides" attributed is copied from the original target
-    # to the synthetic target,  so that the latter can be used as a direct stand-in for the
-    # former here.
-    preliminary_targets = set(t for t in self.context.target_roots if self.has_provides(t))
-    targets = set(preliminary_targets)
-    for t in self.context.targets():
-      # A non-codegen target has derived_from equal to itself, so we check is_original
-      # to ensure that the synthetic targets take precedence.
-      # We check that the synthetic target has the same "provides" as the original, because
-      # there are other synthetic targets in play (e.g., resources targets) to which this
-      # substitution logic must not apply.
-      if (t.derived_from in preliminary_targets and not t.is_original and
-          self.has_provides(t) and t.provides == t.derived_from.provides):
-        targets.discard(t.derived_from)
-        targets.add(t)
-    if not targets:
+    # We drive creation of setup.py distributions from the original target graph, grabbing codegen'd
+    # sources when needed.
+    def is_exported_python_target(t):
+      return t.is_original and self.has_provides(t)
+
+    exported_python_targets = OrderedSet(t for t in self.context.target_roots
+                                         if is_exported_python_target(t))
+    if not exported_python_targets:
       raise TaskError('setup-py target(s) must provide an artifact.')
 
     dist_dir = self.get_options().pants_distdir
@@ -605,23 +608,24 @@ def execute(self):
 
     created = {}
 
-    def create(target):
-      if target not in created:
-        self.context.log.info('Creating setup.py project for {}'.format(target))
-        setup_dir, dependencies = self.create_setup_py(target, dist_dir)
-        created[target] = setup_dir
+    def create(exported_python_target):
+      if exported_python_target not in created:
+        self.context.log.info('Creating setup.py project for {}'.format(exported_python_target))
+        subject = self.derived_by_original.get(exported_python_target, exported_python_target)
+        setup_dir, dependencies = self.create_setup_py(subject, dist_dir)
+        created[exported_python_target] = setup_dir
         if self._recursive:
           for dep in dependencies:
-            if self.has_provides(dep):
+            if is_exported_python_target(dep):
               create(dep)
 
-    for target in targets:
-      create(target)
+    for exported_python_target in exported_python_targets:
+      create(exported_python_target)
 
     interpreter = self.context.products.get_data(PythonInterpreter)
     python_dists = self.context.products.register_data(self.PYTHON_DISTS_PRODUCT, {})
-    for target in reversed(sort_targets(created.keys())):
-      setup_dir = created.get(target)
+    for exported_python_target in reversed(sort_targets(created.keys())):
+      setup_dir = created.get(exported_python_target)
       if setup_dir:
         if not self._run:
           self.context.log.info('Running packager against {}'.format(setup_dir))
@@ -631,9 +635,9 @@ def create(target):
           self.context.log.info('Writing {}'.format(sdist_path))
           shutil.move(setup_runner.sdist(), sdist_path)
           safe_rmtree(setup_dir)
-          python_dists[target] = sdist_path
+          python_dists[exported_python_target] = sdist_path
         else:
           self.context.log.info('Running {} against {}'.format(self._run, setup_dir))
           setup_runner = SetupPyRunner(setup_dir, self._run, interpreter=interpreter)
           setup_runner.run()
-          python_dists[target] = setup_dir
+          python_dists[exported_python_target] = setup_dir


Test patch:
diff --git a/tests/python/pants_test/backend/codegen/thrift/python/BUILD b/tests/python/pants_test/backend/codegen/thrift/python/BUILD
--- a/tests/python/pants_test/backend/codegen/thrift/python/BUILD
+++ b/tests/python/pants_test/backend/codegen/thrift/python/BUILD
@@ -3,6 +3,7 @@
 
 python_tests(
   dependencies=[
+    '3rdparty/python:six',
     'src/python/pants/backend/codegen/thrift/python',
     'src/python/pants/backend/python/targets:python',
     'tests/python/pants_test/tasks:task_test_base',
diff --git a/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py b/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py
--- a/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py
+++ b/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py
@@ -5,8 +5,11 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import os
 from textwrap import dedent
 
+import six
+
 from pants.backend.codegen.thrift.python.apache_thrift_py_gen import ApacheThriftPyGen
 from pants.backend.codegen.thrift.python.python_thrift_library import PythonThriftLibrary
 from pants.backend.python.targets.python_library import PythonLibrary
@@ -31,9 +34,29 @@ def is_synthetic_python_library(target):
     self.assertEqual(1, len(synthetic_targets))
     return synthetic_targets[0]
 
+  def init_py_path(self, target, package_rel_dir):
+    return os.path.join(self.build_root, target.target_base, package_rel_dir, '__init__.py')
+
+  def assert_ns_package(self, target, package_rel_dir):
+    with open(self.init_py_path(target, package_rel_dir)) as fp:
+      self.assertEqual(b"__import__('pkg_resources').declare_namespace(__name__)",
+                       fp.read().strip())
+
+  def assert_leaf_package(self, target, package_rel_dir, *services):
+    # We know thrift controls exported package symbols using `__all__`; so reading this out of the
+    # `__init__.py` is enough to show we haven't trampled non-trivial thrift-generated `__init__.py`
+    # files.
+
+    symbols = {}
+    with open(self.init_py_path(target, package_rel_dir), 'rb') as fp:
+      six.exec_(fp.read(), symbols)
+
+    self.assertIn('__all__', symbols)
+    self.assertEqual(sorted(('constants', 'ttypes') + services), sorted(symbols['__all__']))
+
   def test_single_namespace(self):
     self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
-    namespace py foo
+    namespace py foo.bar
 
     const i32 THINGCONSTANT = 42
 
@@ -45,18 +68,24 @@ def test_single_namespace(self):
                            target_type=PythonThriftLibrary,
                            sources=['one.thrift'])
     synthetic_target = self.generate_single_thrift_target(one)
-    self.assertEqual({'foo/__init__.py', 'foo/ThingService-remote',
-                      'foo/ThingService.py', 'foo/ttypes.py', 'foo/constants.py'},
+    self.assertEqual({'foo/__init__.py',
+                      'foo/bar/__init__.py',
+                      'foo/bar/ThingService-remote',
+                      'foo/bar/ThingService.py',
+                      'foo/bar/ttypes.py',
+                      'foo/bar/constants.py'},
                      set(synthetic_target.sources_relative_to_source_root()))
+    self.assert_ns_package(synthetic_target, 'foo')
+    self.assert_leaf_package(synthetic_target, 'foo/bar', 'ThingService')
 
   def test_nested_namespaces(self):
     self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
-    namespace py foo
+    namespace py foo.bar
 
     struct One {}
     """))
     self.create_file('src/thrift/com/foo/bar/two.thrift', contents=dedent("""
-    namespace py foo.bar
+    namespace py foo.bar.baz
 
     struct Two {}
     """))
@@ -64,6 +93,14 @@ def test_nested_namespaces(self):
                            target_type=PythonThriftLibrary,
                            sources=['one.thrift', 'bar/two.thrift'])
     synthetic_target = self.generate_single_thrift_target(one)
-    self.assertEqual({'foo/__init__.py', 'foo/ttypes.py', 'foo/constants.py',
-                      'foo/bar/__init__.py', 'foo/bar/ttypes.py', 'foo/bar/constants.py'},
+    self.assertEqual({'foo/__init__.py',
+                      'foo/bar/__init__.py',
+                      'foo/bar/constants.py',
+                      'foo/bar/ttypes.py',
+                      'foo/bar/baz/__init__.py',
+                      'foo/bar/baz/constants.py',
+                      'foo/bar/baz/ttypes.py'},
                      set(synthetic_target.sources_relative_to_source_root()))
+    self.assert_ns_package(synthetic_target, 'foo')
+    self.assert_leaf_package(synthetic_target, 'foo/bar')
+    self.assert_leaf_package(synthetic_target, 'foo/bar/baz')
diff --git a/tests/python/pants_test/backend/python/tasks2/test_setup_py_integration.py b/tests/python/pants_test/backend/python/tasks2/test_setup_py_integration.py
--- a/tests/python/pants_test/backend/python/tasks2/test_setup_py_integration.py
+++ b/tests/python/pants_test/backend/python/tasks2/test_setup_py_integration.py
@@ -13,47 +13,122 @@
 
 class SetupPyIntegrationTest(PantsRunIntegrationTest):
 
-  def test_setup_py_with_codegen(self):
-    self.maxDiff = None
+  def assert_sdist(self, pants_run, key, files):
+    sdist_path = 'dist/{}-0.0.1.tar.gz'.format(key)
+    self.assertTrue(re.search(r'Writing .*/{}'.format(sdist_path), pants_run.stdout_data))
 
-    sdist_path = 'dist/pantsbuild.pants.distance-thrift-python-0.0.1.tar.gz'
+    src_entries = ['src/{}'.format(f) for f in files]
+
+    egg_info_entries = [
+      'src/{}.egg-info/{}'.format(key.replace('-', '_'), relpath) for relpath in [
+        '',
+        'dependency_links.txt',
+        'PKG-INFO',
+        'requires.txt',
+        'SOURCES.txt',
+        'namespace_packages.txt',
+        'top_level.txt',
+      ]
+    ]
+
+    expected_entries = [
+      '{}-0.0.1/{}'.format(key, relpath) for relpath in [
+        '',
+        'MANIFEST.in',
+        'PKG-INFO',
+        'setup.cfg',
+        'setup.py',
+        'src/',
+      ] + src_entries + egg_info_entries
+    ]
+
+    with tarfile.open(sdist_path, 'r') as sdist:
+      infos = sdist.getmembers()
+      entries = [(info.name.rstrip('/') + '/' if info.isdir() else info.name) for info in infos]
+      self.assertEquals(sorted(expected_entries), sorted(entries),
+                        '\nExpected entries:\n{}\n\nActual entries:\n{}'.format(
+                          '\n'.join(sorted(expected_entries)),
+                          '\n'.join(sorted(entries))))
+
+  def test_setup_py_with_codegen_simple(self):
+    self.maxDiff = None
 
     command = ['setup-py',
                'examples/src/thrift/org/pantsbuild/example/distance:distance-python']
     pants_run = self.run_pants(command=command)
     self.assert_success(pants_run)
-    self.assertTrue(re.search(r'Writing .*/{}'.format(sdist_path), pants_run.stdout_data))
-    with tarfile.open(sdist_path, 'r') as sdist:
-      entries = sdist.getnames()
-    print(entries)
-    expected_prefix = 'pantsbuild.pants.distance-thrift-python-0.0.1'
-    expected_entries = [
-        expected_prefix + relpath for relpath in [
-        '',
-        '/MANIFEST.in',
-        '/PKG-INFO',
-        '/setup.cfg',
-        '/setup.py',
-        '/src',
-        '/src/org',
-        '/src/org/__init__.py',
-        '/src/org/pantsbuild',
-        '/src/org/pantsbuild/__init__.py',
-        '/src/org/pantsbuild/example',
-        '/src/org/pantsbuild/example/__init__.py',
-        '/src/org/pantsbuild/example/distance',
-        '/src/org/pantsbuild/example/distance/__init__.py',
-        '/src/org/pantsbuild/example/distance/constants.py',
-        '/src/org/pantsbuild/example/distance/ttypes.py',
-        '/src/pantsbuild.pants.distance_thrift_python.egg-info',
-        '/src/pantsbuild.pants.distance_thrift_python.egg-info/dependency_links.txt',
-        '/src/pantsbuild.pants.distance_thrift_python.egg-info/PKG-INFO',
-        '/src/pantsbuild.pants.distance_thrift_python.egg-info/requires.txt',
-        '/src/pantsbuild.pants.distance_thrift_python.egg-info/SOURCES.txt',
-        '/src/pantsbuild.pants.distance_thrift_python.egg-info/top_level.txt'
-      ]
+
+    self.assert_sdist(pants_run,
+                      'pantsbuild.pants.distance-thrift-python',
+                      ['org/',
+                       'org/__init__.py',
+                       'org/pantsbuild/',
+                       'org/pantsbuild/__init__.py',
+                       'org/pantsbuild/example/',
+                       'org/pantsbuild/example/__init__.py',
+                       'org/pantsbuild/example/distance/',
+                       'org/pantsbuild/example/distance/__init__.py',
+                       'org/pantsbuild/example/distance/constants.py',
+                       'org/pantsbuild/example/distance/ttypes.py'])
+
+  def test_setup_py_with_codegen_exported_deps(self):
+    self.maxDiff = None
+
+    command = ['setup-py',
+               '--recursive',
+               'examples/src/thrift/org/pantsbuild/example/precipitation:precipitation-python']
+    pants_run = self.run_pants(command=command)
+    self.assert_success(pants_run)
+
+    self.assert_sdist(pants_run,
+                      'pantsbuild.pants.precipitation-thrift-python',
+                      ['org/',
+                       'org/__init__.py',
+                       'org/pantsbuild/',
+                       'org/pantsbuild/__init__.py',
+                       'org/pantsbuild/example/',
+                       'org/pantsbuild/example/__init__.py',
+                       'org/pantsbuild/example/precipitation/',
+                       'org/pantsbuild/example/precipitation/__init__.py',
+                       'org/pantsbuild/example/precipitation/constants.py',
+                       'org/pantsbuild/example/precipitation/ttypes.py'])
+
+    self.assert_sdist(pants_run,
+                      'pantsbuild.pants.distance-thrift-python',
+                      ['org/',
+                       'org/__init__.py',
+                       'org/pantsbuild/',
+                       'org/pantsbuild/__init__.py',
+                       'org/pantsbuild/example/',
+                       'org/pantsbuild/example/__init__.py',
+                       'org/pantsbuild/example/distance/',
+                       'org/pantsbuild/example/distance/__init__.py',
+                       'org/pantsbuild/example/distance/constants.py',
+                       'org/pantsbuild/example/distance/ttypes.py'])
+
+  def test_setup_py_with_codegen_unexported_deps(self):
+    self.maxDiff = None
+
+    command = [
+      'setup-py',
+      'examples/src/thrift/org/pantsbuild/example/precipitation:monolithic-precipitation-python'
     ]
-    self.assertEquals(
-      sorted(expected_entries),
-      sorted(entries)
-    )
+    pants_run = self.run_pants(command=command)
+    self.assert_success(pants_run)
+
+    self.assert_sdist(pants_run,
+                      'pantsbuild.pants.monolithic-precipitation-thrift-python',
+                      ['org/',
+                       'org/__init__.py',
+                       'org/pantsbuild/',
+                       'org/pantsbuild/__init__.py',
+                       'org/pantsbuild/example/',
+                       'org/pantsbuild/example/__init__.py',
+                       'org/pantsbuild/example/distance/',
+                       'org/pantsbuild/example/distance/__init__.py',
+                       'org/pantsbuild/example/distance/constants.py',
+                       'org/pantsbuild/example/distance/ttypes.py',
+                       'org/pantsbuild/example/precipitation/',
+                       'org/pantsbuild/example/precipitation/__init__.py',
+                       'org/pantsbuild/example/precipitation/constants.py',
+                       'org/pantsbuild/example/precipitation/ttypes.py'])