Repository: pantsbuild/pants

Issue:
setup-py loads __init__.py with errors
Hi,

I'm evaluating pants and it's looking awesome so far! However it seems I run into a possible issue. I'm using python 3.6 and noticed that when running `./pants setup-py` I get an error message like `Exception message: invalid syntax (__init__.py, line 5)`. I'm using pants 1.8.0. 

While investigating, I noticed that if I disable some python 3 features like type hinting and f-strings I don't get the error. Given that both these features were introduced in later python versions, is it possible that pants uses system interpreter (or a wrong interpreter) for loading `__init__.py`?

I created a repo to reproduce the problem: https://github.com/ifoukarakis/pants-typehints. 


Discussion:
Turning on stacktraces:
```
jsirois@gill ~/dev/ifoukarakis/pants-typehints (master) $ ./pants --print-exception-stacktrace setup-py src/python/package:my-package

09:51:16 00:00 [main]
               (To run a reporting server: ./pants server)
09:51:16 00:00   [setup]
09:51:16 00:00     [parse]
               Executing tasks in goals: bootstrap -> imports -> unpack-jars -> deferred-sources -> gen -> pyprep -> setup-py
09:51:16 00:00   [bootstrap]
09:51:16 00:00     [substitute-aliased-targets]
09:51:16 00:00     [jar-dependency-management]
09:51:17 00:01     [bootstrap-jvm-tools]
09:51:17 00:01     [provide-tools-jar]
09:51:17 00:01   [imports]
09:51:17 00:01     [ivy-imports]
09:51:17 00:01   [unpack-jars]
09:51:17 00:01     [unpack-jars]
09:51:17 00:01   [deferred-sources]
09:51:17 00:01     [deferred-sources]
09:51:17 00:01   [gen]
09:51:17 00:01     [antlr-java]
09:51:17 00:01     [antlr-py]
09:51:17 00:01     [jaxb]
09:51:17 00:01     [protoc]
09:51:17 00:01     [ragel]
09:51:17 00:01     [thrift-java]
09:51:17 00:01     [thrift-py]
09:51:17 00:01     [wire]
09:51:17 00:01   [pyprep]
09:51:17 00:01     [interpreter]
09:51:17 00:01     [build-local-dists]
09:51:17 00:01     [requirements]
09:51:17 00:01     [sources]
09:51:17 00:01   [setup-py]
09:51:17 00:01     [setup-py]
                   Creating setup.py project for PythonLibrary(BuildFileAddress(src/python/package/BUILD, my-package))
               Waiting for background workers to finish.
09:51:17 00:01   [complete]
               FAILURE
Exception caught: (<type 'exceptions.SyntaxError'>)
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/bin/pants", line 11, in <module>
    sys.exit(main())
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/bin/pants_loader.py", line 69, in main
    PantsLoader.run()
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/bin/pants_loader.py", line 65, in run
    cls.load_and_execute(entrypoint)
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/bin/pants_loader.py", line 58, in load_and_execute
    entrypoint_main()
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/bin/pants_exe.py", line 39, in main
    PantsRunner(exiter, start_time=start_time).run()
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/bin/pants_runner.py", line 53, in run
    return runner.run()
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/bin/local_pants_runner.py", line 46, in run
    self._run()
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/bin/local_pants_runner.py", line 94, in _run
    goal_runner_result = goal_runner.run()
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/bin/goal_runner.py", line 222, in run
    result = self._execute_engine()
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/bin/goal_runner.py", line 210, in _execute_engine
    result = engine.execute(self._context, self._goals)
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/engine/legacy_engine.py", line 26, in execute
    self.attempt(context, goals)
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/engine/round_engine.py", line 233, in attempt
    goal_executor.attempt(explain)
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/engine/round_engine.py", line 49, in attempt
    task.execute()
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/backend/python/tasks/setup_py.py", line 644, in execute
    create(exported_python_target)
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/backend/python/tasks/setup_py.py", line 636, in create
    setup_dir, dependencies = self.create_setup_py(subject, dist_dir)
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/backend/python/tasks/setup_py.py", line 605, in create_setup_py
    self.write_setup(target, reduced_deps, chroot)
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/backend/python/tasks/setup_py.py", line 539, in write_setup
    packages, namespace_packages, resources = self.find_packages(chroot, self.context.log)
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/backend/python/tasks/setup_py.py", line 444, in find_packages
    if cls.declares_namespace_package(real_filename):
  File "/home/jsirois/.cache/pants/setup/bootstrap-Linux-x86_64/1.8.0/lib/python2.7/site-packages/pants/backend/python/tasks/setup_py.py", line 397, in declares_namespace_package
    init_py = ast.parse(fp.read(), filename)
  File "/usr/lib64/python2.7/ast.py", line 37, in parse
    return compile(source, filename, mode, PyCF_ONLY_AST)

Exception message: invalid syntax (__init__.py, line 5)
```

Resolving patch:
diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py
--- a/src/python/pants/backend/python/tasks/setup_py.py
+++ b/src/python/pants/backend/python/tasks/setup_py.py
@@ -4,11 +4,12 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import ast
+import inspect
 import itertools
 import os
 import pprint
 import shutil
+import textwrap
 from abc import abstractmethod
 from builtins import map, object, str, zip
 from collections import OrderedDict, defaultdict
@@ -16,6 +17,9 @@
 from future.utils import PY2, PY3
 from pex.installer import InstallerBase, Packager
 from pex.interpreter import PythonInterpreter
+from pex.pex import PEX
+from pex.pex_builder import PEXBuilder
+from pex.pex_info import PexInfo
 from twitter.common.collections import OrderedSet
 from twitter.common.dirutil.chroot import Chroot
 
@@ -26,13 +30,16 @@
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TargetDefinitionException, TaskError
 from pants.base.specs import SiblingAddresses
+from pants.base.workunit import WorkUnitLabel
 from pants.build_graph.address_lookup_error import AddressLookupError
 from pants.build_graph.build_graph import sort_targets
 from pants.build_graph.resources import Resources
 from pants.task.task import Task
-from pants.util.dirutil import safe_rmtree, safe_walk
+from pants.util.contextutil import temporary_file
+from pants.util.dirutil import safe_concurrent_creation, safe_rmtree, safe_walk
 from pants.util.memo import memoized_property
 from pants.util.meta import AbstractClass
+from pants.util.process_handler import subprocess
 from pants.util.strutil import safe_shlex_split
 
 
@@ -297,6 +304,26 @@ def collect_reduced_dependencies(current):
     return OrderedSet(d for d in reduced_dependencies if d.is_original)
 
 
+def declares_namespace_package(filename):
+  """Given a filename, walk its ast and determine if it declares a namespace package."""
+
+  import ast
+
+  with open(filename) as fp:
+    init_py = ast.parse(fp.read(), filename)
+  calls = [node for node in ast.walk(init_py) if isinstance(node, ast.Call)]
+  for call in calls:
+    if len(call.args) != 1:
+      continue
+    if isinstance(call.func, ast.Attribute) and call.func.attr != 'declare_namespace':
+      continue
+    if isinstance(call.func, ast.Name) and call.func.id != 'declare_namespace':
+      continue
+    if isinstance(call.args[0], ast.Name) and call.args[0].id == '__name__':
+      return True
+  return False
+
+
 class SetupPy(Task):
   """Generate setup.py-based Python projects."""
 
@@ -366,26 +393,6 @@ def iter_entry_points(cls, target):
             'Cannot add a binary to a PythonArtifact if it does not contain an entry_point.')
       yield name, concrete_target.entry_point
 
-  @classmethod
-  def declares_namespace_package(cls, filename):
-    """Given a filename, walk its ast and determine if it is declaring a namespace package.
-
-    Intended only for __init__.py files though it will work for any .py.
-    """
-    with open(filename) as fp:
-      init_py = ast.parse(fp.read(), filename)
-    calls = [node for node in ast.walk(init_py) if isinstance(node, ast.Call)]
-    for call in calls:
-      if len(call.args) != 1:
-        continue
-      if isinstance(call.func, ast.Attribute) and call.func.attr != 'declare_namespace':
-        continue
-      if isinstance(call.func, ast.Name) and call.func.id != 'declare_namespace':
-        continue
-      if isinstance(call.args[0], ast.Name) and call.args[0].id == '__name__':
-        return True
-    return False
-
   @classmethod
   def nearest_subpackage(cls, package, all_packages):
     """Given a package, find its nearest parent in all_packages."""
@@ -396,8 +403,66 @@ def shared_prefix(candidate):
     shared_packages = [_f for _f in map(shared_prefix, all_packages) if _f]
     return '.'.join(max(shared_packages, key=len)) if shared_packages else package
 
-  @classmethod
-  def find_packages(cls, chroot, log=None):
+  @memoized_property
+  def nsutil_pex(self):
+    interpreter = self.context.products.get_data(PythonInterpreter)
+    chroot = os.path.join(self.workdir, 'nsutil', interpreter.version_string)
+    if not os.path.exists(chroot):
+      pex_info = PexInfo.default(interpreter=interpreter)
+      with safe_concurrent_creation(chroot) as scratch:
+        builder = PEXBuilder(path=scratch, interpreter=interpreter, pex_info=pex_info, copy=True)
+        with temporary_file() as fp:
+          declares_namespace_package_code = inspect.getsource(declares_namespace_package)
+          fp.write(textwrap.dedent('''
+            import sys
+
+
+            {declares_namespace_package_code}
+
+
+            if __name__ == '__main__':
+              for path in sys.argv[1:]:
+                if declares_namespace_package(path):
+                  print(path)
+          ''').strip().format(declares_namespace_package_code=declares_namespace_package_code))
+          fp.close()
+          builder.set_executable(filename=fp.name, env_filename='main.py')
+          builder.freeze()
+    return PEX(pex=chroot, interpreter=interpreter)
+
+  def filter_namespace_packages(self, root_target, inits):
+    args = list(inits)
+    with self.context.new_workunit(name='find-namespace-packages',
+                                   cmd=' '.join(self.nsutil_pex.cmdline(args=args)),
+                                   labels=[WorkUnitLabel.TOOL]) as workunit:
+
+      process = self.nsutil_pex.run(args=args,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE,
+                                    blocking=False)
+
+      stdout, stderr = process.communicate()
+
+      # TODO(John Sirois): Find a way to tee a workunit output instead of buffering up all output
+      # and then writing it out after the process has finished like we do here.
+      def write(stream_name, data):
+        stream = workunit.output(stream_name)
+        stream.write(data)
+        stream.flush()
+
+      write('stdout', stdout)
+      write('stderr', stderr)
+
+      exit_code = process.returncode
+      if exit_code != 0:
+        raise TaskError('Failure trying to detect namespace packages when constructing setup.py '
+                        'project for {}:\n{}'.format(root_target.address.reference(), stderr),
+                        exit_code=exit_code,
+                        failed_targets=[root_target])
+
+      return stdout.splitlines()
+
+  def find_packages(self, root_target, chroot):
     """Detect packages, namespace packages and resources from an existing chroot.
 
     :returns: a tuple of:
@@ -405,7 +470,7 @@ def find_packages(cls, chroot, log=None):
                 set(namespace_packages)
                 map(package => set(files))
     """
-    base = os.path.join(chroot.path(), cls.SOURCE_ROOT)
+    base = os.path.join(chroot.path(), self.SOURCE_ROOT)
     packages, namespace_packages = set(), set()
     resources = defaultdict(set)
 
@@ -416,12 +481,15 @@ def iter_files():
           yield module, filename, os.path.join(root, filename)
 
     # establish packages, namespace packages in first pass
+    inits_to_check = {}
     for module, filename, real_filename in iter_files():
       if filename != '__init__.py':
         continue
       packages.add(module)
-      if cls.declares_namespace_package(real_filename):
-        namespace_packages.add(module)
+      inits_to_check[real_filename] = module
+    namespace_packages = {inits_to_check[init]
+                          for init in self.filter_namespace_packages(root_target,
+                                                                     inits_to_check.keys())}
 
     # second pass establishes non-source content (resources)
     for module, filename, real_filename in iter_files():
@@ -431,11 +499,11 @@ def iter_files():
           # hygiene.
           # raise cls.UndefinedSource('{} is source but does not belong to a package!'
           #                           .format(filename))
-          if log:
-            log.warn('{} is source but does not belong to a package.'.format(real_filename))
+          self.context.log.warn('{} is source but does not belong to a package.'
+                                .format(real_filename))
         else:
           continue
-      submodule = cls.nearest_subpackage(module, packages)
+      submodule = self.nearest_subpackage(module, packages)
       if submodule == module:
         resources[submodule].add(filename)
       else:
@@ -512,14 +580,15 @@ def write_setup(self, root_target, reduced_dependencies, chroot):
     # NB: several explicit str conversions below force non-unicode strings in order to comply
     # with setuptools expectations.
     #
-    # Because we rely on pprint to generate the content, we must treat Py2 and Py3 differently. In Py2, unicode strings
-    # have the prefix 'u', and in Py3 byte strings have the prefix 'b'. The goal is to have no prefix displayed, so to
-    # do this we have to use bytes in Py2 and unicode in Py3.
+    # Because we rely on pprint to generate the content, we must treat Py2 and Py3 differently.
+    # In Py2, unicode strings have the prefix 'u', and in Py3 byte strings have the prefix 'b'.
+    # The goal is to have no prefix displayed, so to do this we have to use bytes in Py2 and
+    # unicode in Py3.
 
     setup_keywords = root_target.provides.setup_py_keywords.copy()
 
     package_dir = {'' if PY3 else b'': self.SOURCE_ROOT}
-    packages, namespace_packages, resources = self.find_packages(chroot, self.context.log)
+    packages, namespace_packages, resources = self.find_packages(root_target, chroot)
 
     if namespace_packages:
       setup_keywords['namespace_packages'] = list(sorted(namespace_packages))


Test patch:
diff --git a/tests/python/pants_test/backend/python/tasks/BUILD b/tests/python/pants_test/backend/python/tasks/BUILD
--- a/tests/python/pants_test/backend/python/tasks/BUILD
+++ b/tests/python/pants_test/backend/python/tasks/BUILD
@@ -52,19 +52,19 @@ python_tests(
 python_tests(
   sources=globs('test_*.py', exclude=([globs('*_integration.py')] + python_native_code_test_files)),
   dependencies=[
-    ':python_task_test_base',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
-    '3rdparty/python:future',
     '3rdparty/python:coverage',
+    '3rdparty/python:future',
     '3rdparty/python:mock',
     '3rdparty/python:pex',
+    ':python_task_test_base',
     'src/python/pants/backend/native',
-    'src/python/pants/backend/python:plugin',
     'src/python/pants/backend/python/subsystems',
     'src/python/pants/backend/python/targets',
     'src/python/pants/backend/python/tasks',
     'src/python/pants/backend/python:interpreter_cache',
+    'src/python/pants/backend/python:plugin',
     'src/python/pants/backend/python:python_artifact',
     'src/python/pants/backend/python:python_requirement',
     'src/python/pants/base:build_root',
@@ -76,10 +76,11 @@ python_tests(
     'src/python/pants/util:contextutil',
     'src/python/pants/util:dirutil',
     'src/python/pants/util:process_handler',
+    'tests/python/pants_test/backend/python:interpreter_selection_utils',
     'tests/python/pants_test/engine:scheduler_test_base',
     'tests/python/pants_test/subsystem:subsystem_utils',
-    'tests/python/pants_test:task_test_base',
     'tests/python/pants_test/tasks:task_test_base',
+    'tests/python/pants_test:task_test_base',
   ],
   timeout=600
 )
diff --git a/tests/python/pants_test/backend/python/tasks/test_setup_py.py b/tests/python/pants_test/backend/python/tasks/test_setup_py.py
--- a/tests/python/pants_test/backend/python/tasks/test_setup_py.py
+++ b/tests/python/pants_test/backend/python/tasks/test_setup_py.py
@@ -16,8 +16,9 @@
 from twitter.common.dirutil.chroot import Chroot
 
 from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.backend.python.targets.python_library import PythonLibrary
 from pants.backend.python.tasks.select_interpreter import SelectInterpreter
-from pants.backend.python.tasks.setup_py import SetupPy
+from pants.backend.python.tasks.setup_py import SetupPy, declares_namespace_package
 from pants.base.exceptions import TaskError
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.build_graph.prep_command import PrepCommand
@@ -26,6 +27,7 @@
 from pants.fs.archive import TGZ
 from pants.util.contextutil import environment_as, temporary_dir, temporary_file
 from pants.util.dirutil import safe_mkdir
+from pants_test.backend.python.interpreter_selection_utils import skip_unless_python36
 from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase
 from pants_test.subsystem.subsystem_util import init_subsystem
 
@@ -41,15 +43,18 @@ def setUp(self):
     self.set_options(pants_distdir=self.distdir)
     init_subsystem(Target.Arguments)
 
-  @contextmanager
-  def run_execute(self, target, recursive=False):
+  def prepare_setup_py(self, target, recursive=False):
     self.set_options(recursive=recursive)
     si_task_type = self.synthesize_task_subtype(SelectInterpreter, 'si_scope')
     context = self.context(for_task_types=[si_task_type], target_roots=[target])
     si_task_type(context, os.path.join(self.pants_workdir, 'si')).execute()
-    setup_py = self.create_task(context)
+    return self.create_task(context)
+
+  @contextmanager
+  def run_execute(self, target, recursive=False):
+    setup_py = self.prepare_setup_py(target, recursive=recursive)
     setup_py.execute()
-    yield context.products.get_data(SetupPy.PYTHON_DISTS_PRODUCT)
+    yield setup_py.context.products.get_data(SetupPy.PYTHON_DISTS_PRODUCT)
 
 
 class TestSetupPyInterpreter(SetupPyTestBase):
@@ -604,7 +609,7 @@ def has_ns(stmt):
     with temporary_file(binary_mode=False) as fp:
       fp.write(stmt)
       fp.flush()
-      return SetupPy.declares_namespace_package(fp.name)
+      return declares_namespace_package(fp.name)
 
   assert not has_ns('')
   assert not has_ns('add(1, 2); foo(__name__); self.shoot(__name__)')
@@ -614,70 +619,80 @@ def has_ns(stmt):
   assert has_ns('from pkg_resources import declare_namespace; declare_namespace(__name__)')
 
 
-@contextmanager
-def yield_chroot(packages, namespace_packages, resources):
-  def to_path(package):
-    return package.replace('.', os.path.sep)
-
-  with temporary_dir() as td:
-    def write(package, name, content):
-      package_path = os.path.join(td, SetupPy.SOURCE_ROOT, to_path(package))
-      safe_mkdir(os.path.dirname(os.path.join(package_path, name)))
-      with open(os.path.join(package_path, name), 'w') as fp:
-        fp.write(content)
-    for package in packages:
-      write(package, '__init__.py', '')
-    for package in namespace_packages:
-      write(package, '__init__.py', '__import__("pkg_resources").declare_namespace(__name__)')
-    for package, resource_list in resources.items():
-      for resource in resource_list:
-        write(package, resource, 'asdfasdf')
-
-    chroot_mock = Mock(spec=Chroot)
-    chroot_mock.path.return_value = td
-    yield chroot_mock
-
-
-def test_find_packages():
-  def assert_single_chroot(packages, namespace_packages, resources):
-    with yield_chroot(packages, namespace_packages, resources) as chroot:
-      p, n_p, r = SetupPy.find_packages(chroot)
-      assert p == set(packages + namespace_packages)
-      assert n_p == set(namespace_packages)
-      assert r == dict((k, set(v)) for (k, v) in resources.items())
-
-  # assert both packages and namespace packages work
-  assert_single_chroot(['foo'], [], {})
-  assert_single_chroot(['foo'], ['foo'], {})
-
-  # assert resources work
-  assert_single_chroot(['foo'], [], {'foo': ['blork.dat']})
-
-  resources = {
-    'foo': [
-      'f0',
-      os.path.join('bar', 'baz', 'f1'),
-      os.path.join('bar', 'baz', 'f2'),
-    ]
-  }
-  assert_single_chroot(['foo'], [], resources)
-
-  # assert that nearest-submodule is honored
-  with yield_chroot(['foo', 'foo.bar'], [], resources) as chroot:
-    _, _, r = SetupPy.find_packages(chroot)
-    assert r == {
-      'foo': {'f0'},
-      'foo.bar': {os.path.join('baz', 'f1'), os.path.join('baz', 'f2')}
+class TestSetupPyFindPackages(SetupPyTestBase):
+  @contextmanager
+  def yield_chroot(self, packages, namespace_packages, resources, py3=False):
+    def to_path(package):
+      return package.replace('.', os.path.sep)
+
+    with temporary_dir() as td:
+      def write(package, name, content):
+        package_path = os.path.join(td, SetupPy.SOURCE_ROOT, to_path(package))
+        safe_mkdir(os.path.dirname(os.path.join(package_path, name)))
+        with open(os.path.join(package_path, name), 'w') as fp:
+          fp.write(content)
+      for package in packages:
+        write(package, '__init__.py', 'a = "b"; b = f"{a}"' if py3 else '')
+      for package in namespace_packages:
+        write(package, '__init__.py', '__import__("pkg_resources").declare_namespace(__name__)')
+      for package, resource_list in resources.items():
+        for resource in resource_list:
+          write(package, resource, 'asdfasdf')
+
+      chroot_mock = Mock(spec=Chroot)
+      chroot_mock.path.return_value = td
+      yield chroot_mock
+
+  def assert_find_packages(self, py36):
+    compatibility = 'CPython>=3.6,<3.7' if py36 else 'CPython>=2.7,<3'
+    dummy_target = self.make_target(spec='src/python/foo',
+                                    target_type=PythonLibrary,
+                                    sources=[],
+                                    compatibility=compatibility)
+    setup_py = self.prepare_setup_py(dummy_target)
+
+    def assert_single_chroot(packages, namespace_packages, resources):
+      with self.yield_chroot(packages, namespace_packages, resources, py3=py36) as chroot:
+        p, n_p, r = setup_py.find_packages(dummy_target, chroot)
+        assert p == set(packages + namespace_packages)
+        assert n_p == set(namespace_packages)
+        assert r == dict((k, set(v)) for (k, v) in resources.items())
+
+    # assert both packages and namespace packages work
+    assert_single_chroot(['foo'], [], {})
+    assert_single_chroot(['foo'], ['foo'], {})
+
+    # assert resources work
+    assert_single_chroot(['foo'], [], {'foo': ['blork.dat']})
+
+    resources = {
+      'foo': [
+        'f0',
+        os.path.join('bar', 'baz', 'f1'),
+        os.path.join('bar', 'baz', 'f2'),
+      ]
     }
-
-  # assert that nearest submodule splits on module prefixes
-  with yield_chroot(
-      ['foo', 'foo.bar'],
-      [],
-      {'foo.bar1': ['f0']}) as chroot:
-
-    _, _, r = SetupPy.find_packages(chroot)
-    assert r == {'foo': {'bar1/f0'}}
+    assert_single_chroot(['foo'], [], resources)
+
+    # assert that nearest-submodule is honored
+    with self.yield_chroot(['foo', 'foo.bar'], [], resources, py3=py36) as chroot:
+      _, _, r = setup_py.find_packages(dummy_target, chroot)
+      assert r == {
+        'foo': {'f0'},
+        'foo.bar': {os.path.join('baz', 'f1'), os.path.join('baz', 'f2')}
+      }
+
+    # assert that nearest submodule splits on module prefixes
+    with self.yield_chroot(['foo', 'foo.bar'], [], {'foo.bar1': ['f0']}, py3=py36) as chroot:
+      _, _, r = setup_py.find_packages(dummy_target, chroot)
+      assert r == {'foo': {'bar1/f0'}}
+
+  def test_find_packages(self):
+    self.assert_find_packages(py36=False)
+
+  @skip_unless_python36
+  def test_find_packages_py3(self):
+    self.assert_find_packages(py36=True)
 
 
 def test_nearest_subpackage():