Repository: pex-tool/pex

Issue:
pypy shards in travisCI can fail with 'too many open files' errors
this needs more investigation, but intermittently only the "pypy" shards in travis will fail arbitrarily with "too many open files" errors.

we had previously pinned to the travis `precise` image because of this, but because of deprecation prompting for that image we'd jumped forward here: https://github.com/pantsbuild/pex/pull/476

an initial pass may want to try some simple `ulimit` tuning.


Discussion:
filed https://github.com/travis-ci/travis-ci/issues/9706 for some upstream help.
topical relief achieved in #513 here: https://travis-ci.org/pantsbuild/pex/builds/390035309

at a cost of ~4x test execution time (~2m -> ~8m) - good enough to tide us over for now.

Resolving patch:
diff --git a/pex/bootstrap.py b/pex/bootstrap.py
new file mode 100644
--- /dev/null
+++ b/pex/bootstrap.py
@@ -0,0 +1,77 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os
+
+
+class Bootstrap(object):
+  """Supports introspection of the PEX bootstrap code."""
+
+  _INSTANCE = None
+
+  @classmethod
+  def locate(cls):
+    """Locates the active PEX bootstrap.
+
+    :rtype: :class:`Bootstrap`
+    """
+    if cls._INSTANCE is None:
+      bootstrap_path = __file__
+      module_import_path = __name__.split('.')
+
+      # For example, our __file__ might be requests.pex/.bootstrap/pex/bootstrap.pyc and our import
+      # path pex.bootstrap; so we walk back through all the module components of our import path to
+      # find the base sys.path entry where we were found (requests.pex/.bootstrap in this example).
+      for _ in module_import_path:
+        bootstrap_path = os.path.dirname(bootstrap_path)
+
+      cls._INSTANCE = cls(sys_path_entry=bootstrap_path)
+    return cls._INSTANCE
+
+  def __init__(self, sys_path_entry):
+    self._sys_path_entry = sys_path_entry
+    self._realpath = os.path.realpath(self._sys_path_entry)
+
+  def demote(self):
+    """Demote the bootstrap code to the end of the `sys.path` so it is found last.
+
+    :return: The list of un-imported bootstrap modules.
+    :rtype: list of :class:`types.ModuleType`
+    """
+    import sys  # Grab a hold of `sys` early since we'll be un-importing our module in this process.
+
+    unimported_modules = []
+    for name, module in reversed(sorted(sys.modules.items())):
+      if self.imported_from_bootstrap(module):
+        unimported_modules.append(sys.modules.pop(name))
+
+    sys.path.remove(self._sys_path_entry)
+    sys.path.append(self._sys_path_entry)
+
+    return unimported_modules
+
+  def imported_from_bootstrap(self, module):
+    """Return ``True`` if the given ``module`` object was imported from bootstrap code.
+
+    :param module: The module to check the provenance of.
+    :type module: :class:`types.ModuleType`
+    :rtype: bool
+    """
+
+    # A vendored module.
+    path = getattr(module, '__file__', None)
+    if path and os.path.realpath(path).startswith(self._realpath):
+      return True
+
+    # A vendored package.
+    path = getattr(module, '__path__', None)
+    if path and any(os.path.realpath(path_item).startswith(self._realpath)
+                    for path_item in path):
+      return True
+
+    return False
+
+  def __repr__(self):
+    return '{cls}(sys_path_entry={sys_path_entry!r})'.format(cls=type(self).__name__,
+                                                             sys_path_entry=self._sys_path_entry)
diff --git a/pex/environment.py b/pex/environment.py
--- a/pex/environment.py
+++ b/pex/environment.py
@@ -3,13 +3,17 @@
 
 from __future__ import absolute_import
 
+import importlib
 import itertools
 import os
 import site
 import sys
 import uuid
+import warnings
+import zipfile
 
 from pex import pex_builder
+from pex.bootstrap import Bootstrap
 from pex.common import die, open_zip, rename_if_empty, safe_mkdir, safe_rmtree
 from pex.interpreter import PythonInterpreter
 from pex.package import distribution_compatible
@@ -26,20 +30,93 @@
 from pex.util import CacheHelper, DistributionHelper
 
 
+def _import_pkg_resources():
+  try:
+    import pkg_resources
+    return pkg_resources, False
+  except ImportError:
+    from pex import third_party
+    third_party.install(expose=['setuptools'])
+    import pkg_resources
+    return pkg_resources, True
+
+
 class PEXEnvironment(Environment):
+  class _CachingZipImporter(object):
+    class _CachingLoader(object):
+      def __init__(self, delegate):
+        self._delegate = delegate
+
+      def load_module(self, fullname):
+        loaded = sys.modules.get(fullname)
+        # Technically a PEP-302 loader should re-load the existing module object here - notably
+        # re-exec'ing the code found in the zip against the existing module __dict__. We don't do
+        # this since the zip is assumed immutable during our run and this is enough to work around
+        # the issue.
+        if not loaded:
+          loaded = self._delegate.load_module(fullname)
+          loaded.__loader__ = self
+        return loaded
+
+    _REGISTERED = False
+
+    @classmethod
+    def _ensure_namespace_handler_registered(cls):
+      if not cls._REGISTERED:
+        pkg_resources, _ = _import_pkg_resources()
+        pkg_resources.register_namespace_handler(cls, pkg_resources.file_ns_handler)
+        cls._REGISTERED = True
+
+    def __init__(self, path):
+      import zipimport
+      self._delegate = zipimport.zipimporter(path)
+
+    def find_module(self, fullname, path=None):
+      loader = self._delegate.find_module(fullname, path)
+      if loader is None:
+        return None
+      self._ensure_namespace_handler_registered()
+      caching_loader = self._CachingLoader(loader)
+      return caching_loader
+
+  @classmethod
+  def _install_pypy_zipimporter_workaround(cls, pex_file):
+    # The pypy zipimporter implementation always freshly loads a module instead of re-importing
+    # when the module already exists in sys.modules. This breaks the PEP-302 importer protocol and
+    # violates pkg_resources assumptions based on that protocol in its handling of namespace
+    # packages. See: https://bitbucket.org/pypy/pypy/issues/1686
+
+    def pypy_zipimporter_workaround(path):
+      import os
+
+      if not path.startswith(pex_file) or '.' in os.path.relpath(path, pex_file):
+        # We only need to claim the pex zipfile root modules.
+        #
+        # The protocol is to raise if we don't want to hook the given path.
+        # See: https://www.python.org/dev/peps/pep-0302/#specification-part-2-registering-hooks
+        raise ImportError()
+
+      return cls._CachingZipImporter(path)
+
+    for path in list(sys.path_importer_cache):
+      if path.startswith(pex_file):
+        sys.path_importer_cache.pop(path)
+
+    sys.path_hooks.insert(0, pypy_zipimporter_workaround)
+
   @classmethod
-  def force_local(cls, pex, pex_info):
+  def force_local(cls, pex_file, pex_info):
     if pex_info.code_hash is None:
       # Do not support force_local if code_hash is not set. (It should always be set.)
-      return pex
+      return pex_file
     explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash)
     TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir)
     if not os.path.exists(explode_dir):
       explode_tmp = explode_dir + '.' + uuid.uuid4().hex
-      with TRACER.timed('Unzipping %s' % pex):
+      with TRACER.timed('Unzipping %s' % pex_file):
         try:
           safe_mkdir(explode_tmp)
-          with open_zip(pex) as pex_zip:
+          with open_zip(pex_file) as pex_zip:
             pex_files = (x for x in pex_zip.namelist()
                          if not x.startswith(pex_builder.BOOTSTRAP_DIR) and
                             not x.startswith(PexInfo.INTERNAL_CACHE))
@@ -52,26 +129,42 @@ def force_local(cls, pex, pex_info):
     return explode_dir
 
   @classmethod
-  def update_module_paths(cls, new_code_path):
-    # Force subsequent imports to come from the .pex directory rather than the .pex file.
-    TRACER.log('Adding to the head of sys.path: %s' % new_code_path)
-    sys.path.insert(0, new_code_path)
-    for name, module in sys.modules.items():
-      if hasattr(module, '__path__'):
-        module_dir = os.path.join(new_code_path, *name.split("."))
-        TRACER.log('Adding to the head of %s.__path__: %s' % (module.__name__, module_dir))
-        try:
-          module.__path__.insert(0, module_dir)
-        except AttributeError:
-          # TODO: This is a temporary bandaid for an unhandled AttributeError which results
-          # in a startup crash. See https://github.com/pantsbuild/pex/issues/598 for more info.
-          TRACER.log(
-            'Failed to insert %s: %s.__path__ of type %s does not support insertion!' % (
-              module_dir,
-              module.__name__,
-              type(module.__path__)
-            )
-          )
+  def update_module_paths(cls, pex_file, explode_dir):
+    bootstrap = Bootstrap.locate()
+    pex_path = os.path.realpath(pex_file)
+
+    # Un-import any modules already loaded from within the .pex file.
+    to_reimport = []
+    for name, module in reversed(sorted(sys.modules.items())):
+      if bootstrap.imported_from_bootstrap(module):
+        TRACER.log('Not re-importing module %s from bootstrap.' % module, V=3)
+        continue
+
+      pkg_path = getattr(module, '__path__', None)
+      if pkg_path and any(os.path.realpath(path_item).startswith(pex_path)
+                          for path_item in pkg_path):
+        sys.modules.pop(name)
+        to_reimport.append((name, pkg_path, True))
+      elif name != '__main__':  # The __main__ module is special in python and is not re-importable.
+        mod_file = getattr(module, '__file__', None)
+        if mod_file and os.path.realpath(mod_file).startswith(pex_path):
+          sys.modules.pop(name)
+          to_reimport.append((name, mod_file, False))
+
+    # Force subsequent imports to come from the exploded .pex directory rather than the .pex file.
+    TRACER.log('Adding to the head of sys.path: %s' % explode_dir)
+    sys.path.insert(0, explode_dir)
+
+    # And re-import them from the exploded pex.
+    for name, existing_path, is_pkg in to_reimport:
+      TRACER.log('Re-importing %s %s loaded via %r from exploded pex.'
+                 % ('package' if is_pkg else 'module', name, existing_path))
+      reimported_module = importlib.import_module(name)
+      if is_pkg:
+        for path_item in existing_path:
+          # NB: It is not guaranteed that __path__ is a list, it may be a PEP-420 namespace package
+          # object which supports a limited mutation API; so we append each item individually.
+          reimported_module.__path__.append(path_item)
 
   @classmethod
   def write_zipped_internal_cache(cls, pex, pex_info):
@@ -131,6 +224,11 @@ def __init__(self, pex, pex_info, interpreter=None, **kw):
     self._inherit_path = pex_info.inherit_path
     self._supported_tags = []
 
+    # For the bug this works around, see: https://bitbucket.org/pypy/pypy/issues/1686
+    # NB: This must be installed early before the underlying pex is loaded in any way.
+    if self._interpreter.identity.abbr_impl == 'pp' and zipfile.is_zipfile(self._pex):
+      self._install_pypy_zipimporter_workaround(self._pex)
+
     platform = Platform.current()
     platform_name = platform.platform
     super(PEXEnvironment, self).__init__(
@@ -209,11 +307,38 @@ def _resolve(self, working_set, reqs):
 
     return resolveds
 
+  @staticmethod
+  def _declare_namespace_packages(dist):
+    if not dist.has_metadata('namespace_packages.txt'):
+      return  # Nothing to do here.
+
+    # NB: Some distributions (notably `twitter.common.*`) in the wild declare setuptools-specific
+    # `namespace_packages` but do not properly declare a dependency on setuptools which they must
+    # use to:
+    # 1. Declare namespace_packages metadata which we just verified they have with the check above.
+    # 2. Declare namespace packages at runtime via the canonical:
+    #    `__import__('pkg_resources').declare_namespace(__name__)`
+    #
+    # As such, we assume the best and try to import pkg_resources from the distribution they depend
+    # on, and then fall back to our vendored version only if not present. This is safe, since we'll
+    # only introduce our shaded version when no other standard version is present and even then tear
+    # it all down when we hand off from the bootstrap to user code.
+    pkg_resources, vendored = _import_pkg_resources()
+    if vendored:
+      warnings.warn('The `pkg_resources` package was loaded from a pex vendored version when '
+                    'declaring namespace packages defined by {dist}. The {dist} distribution '
+                    'should fix its `install_requires` to include `setuptools`'.format(dist=dist))
+
+    for pkg in dist.get_metadata_lines('namespace_packages.txt'):
+      if pkg in sys.modules:
+        pkg_resources.declare_namespace(pkg)
+
   def _activate(self):
     self.update_candidate_distributions(self.load_internal_cache(self._pex, self._pex_info))
 
     if not self._pex_info.zip_safe and os.path.isfile(self._pex):
-      self.update_module_paths(self.force_local(self._pex, self._pex_info))
+      explode_dir = self.force_local(pex_file=self._pex, pex_info=self._pex_info)
+      self.update_module_paths(pex_file=self._pex, explode_dir=explode_dir)
 
     all_reqs = [Requirement.parse(req) for req in self._pex_info.requirements]
 
@@ -224,22 +349,25 @@ def _activate(self):
       with TRACER.timed('Activating %s' % dist, V=2):
         working_set.add(dist)
 
-        if os.path.isdir(dist.location):
-          with TRACER.timed('Adding sitedir', V=2):
-            if dist.location not in sys.path and self._inherit_path == "fallback":
-              # Prepend location to sys.path.
-              # This ensures that bundled versions of libraries will be used before system-installed
-              # versions, in case something is installed in both, helping to favor hermeticity in
-              # the case of non-hermetic PEX files (i.e. those with inherit_path=True).
-              #
-              # If the path is not already in sys.path, site.addsitedir will append (not prepend)
-              # the path to sys.path. But if the path is already in sys.path, site.addsitedir will
-              # leave sys.path unmodified, but will do everything else it would do. This is not part
-              # of its advertised contract (which is very vague), but has been verified to be the
-              # case by inspecting its source for both cpython 2.7 and cpython 3.7.
-              sys.path.insert(0, dist.location)
-            site.addsitedir(dist.location)
-
-        dist.activate()
+        if self._inherit_path == "fallback":
+          # Prepend location to sys.path.
+          #
+          # This ensures that bundled versions of libraries will be used before system-installed
+          # versions, in case something is installed in both, helping to favor hermeticity in
+          # the case of non-hermetic PEX files (i.e. those with inherit_path=True).
+          #
+          # If the path is not already in sys.path, site.addsitedir will append (not prepend)
+          # the path to sys.path. But if the path is already in sys.path, site.addsitedir will
+          # leave sys.path unmodified, but will do everything else it would do. This is not part
+          # of its advertised contract (which is very vague), but has been verified to be the
+          # case by inspecting its source for both cpython 2.7 and cpython 3.7.
+          sys.path.insert(0, dist.location)
+        else:
+          sys.path.append(dist.location)
+
+        with TRACER.timed('Adding sitedir', V=2):
+          site.addsitedir(dist.location)
+
+        self._declare_namespace_packages(dist)
 
     return working_set
diff --git a/pex/pex.py b/pex/pex.py
--- a/pex/pex.py
+++ b/pex/pex.py
@@ -4,13 +4,13 @@
 from __future__ import absolute_import, print_function
 
 import os
-import pkgutil
 import sys
 from distutils import sysconfig
 from site import USER_SITE
 
 import pex.third_party.pkg_resources as pkg_resources
 from pex import third_party
+from pex.bootstrap import Bootstrap
 from pex.common import die
 from pex.environment import PEXEnvironment
 from pex.executor import Executor
@@ -163,7 +163,7 @@ def minimum_sys_modules(cls, site_libs, modules=None):
     metadata such as *.dist-info/namespace_packages.txt.  This can possibly cause namespace
     packages to leak into imports despite being scrubbed from sys.path.
 
-    NOTE: This method mutates modules' __path__ attributes in sys.module, so this is currently an
+    NOTE: This method mutates modules' __path__ attributes in sys.modules, so this is currently an
     irreversible operation.
     """
 
@@ -176,7 +176,7 @@ def minimum_sys_modules(cls, site_libs, modules=None):
         new_modules[module_name] = module
         continue
 
-      # Unexpected objects, e.g. namespace packages, should just be dropped:
+      # Unexpected objects, e.g. PEP 420 namespace packages, should just be dropped.
       if not isinstance(module.__path__, list):
         TRACER.log('Dropping %s' % (module_name,), V=3)
         continue
@@ -403,40 +403,22 @@ def _execute(self):
   def demote_bootstrap(cls):
     TRACER.log('Bootstrap complete, performing final sys.path modifications...')
 
-    bootstrap_path = __file__
-    module_import_path = __name__.split('.')
-
-    # For example, our __file__ might be requests.pex/.bootstrap/pex/pex.pyc and our import path
-    # pex.pex; so we walk back through all the module components of our import path to find the
-    # base sys.path entry where we were found (requests.pex/.bootstrap in this example).
-    for _ in module_import_path:
-      bootstrap_path = os.path.dirname(bootstrap_path)
-    bootstrap_path_index = sys.path.index(bootstrap_path)
-
     should_log = {level: TRACER.should_log(V=level) for level in range(1, 10)}
 
     def log(msg, V=1):
       if should_log.get(V, False):
         print('pex: {}'.format(msg), file=sys.stderr)
 
-    # Move the third party resources pex uses to the end of sys.path for the duration of the run to
-    # allow conflicting versions supplied by user dependencies to win during the course of the
-    # execution of user code.
+    # Remove the third party resources pex uses and demote pex bootstrap code to the end of
+    # sys.path for the duration of the run to allow conflicting versions supplied by user
+    # dependencies to win during the course of the execution of user code.
     unregister_finders()
     third_party.uninstall()
-    for _, mod, _ in pkgutil.iter_modules([bootstrap_path]):
-      if mod in sys.modules:
-        log('Un-importing bootstrap dependency %s from %s' % (mod, bootstrap_path), V=2)
-        sys.modules.pop(mod)
-        log('un-imported {}'.format(mod), V=9)
-
-        submod_prefix = mod + '.'
-        for submod in sorted(m for m in sys.modules.keys() if m.startswith(submod_prefix)):
-          sys.modules.pop(submod)
-          log('un-imported {}'.format(submod), V=9)
-
-    sys.path.pop(bootstrap_path_index)
-    sys.path.append(bootstrap_path)
+
+    bootstrap = Bootstrap.locate()
+    log('Demoting code from %s' % bootstrap, V=2)
+    for module in bootstrap.demote():
+      log('un-imported {}'.format(module), V=9)
 
     import pex
     log('Re-imported pex from {}'.format(pex.__path__), V=3)


Test patch:
diff --git a/tests/test_environment.py b/tests/test_environment.py
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -1,14 +1,16 @@
 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
+
 import os
 import platform
 import subprocess
 from contextlib import contextmanager
+from textwrap import dedent
 
 import pytest
 
 from pex import resolver, vendor
-from pex.compatibility import nested, to_bytes
+from pex.compatibility import PY2, nested, to_bytes
 from pex.environment import PEXEnvironment
 from pex.installer import EggInstaller, WheelInstaller
 from pex.interpreter import PythonInterpreter
@@ -16,7 +18,15 @@
 from pex.pex import PEX
 from pex.pex_builder import PEXBuilder
 from pex.pex_info import PexInfo
-from pex.testing import make_bdist, temporary_dir, temporary_filename
+from pex.resolver import resolve
+from pex.testing import (
+    PY35,
+    ensure_python_interpreter,
+    make_bdist,
+    temporary_content,
+    temporary_dir,
+    temporary_filename
+)
 
 
 @contextmanager
@@ -49,6 +59,143 @@ def test_force_local():
     assert PEXEnvironment.force_local(pex_file, pb.info) == code_cache
 
 
+def assert_force_local_implicit_ns_packages_issues_598(interpreter=None,
+                                                       requirements=(),
+                                                       create_ns_packages=True):
+
+  def create_foo_bar_setup(name, **extra_args):
+    setup_args = dict(
+      name=name,
+      version='0.0.1',
+      packages=['foo', 'foo.bar']
+    )
+    if create_ns_packages:
+      setup_args.update(namespace_packages=['foo', 'foo.bar'])
+    if requirements:
+      setup_args.update(install_requires=list(requirements))
+    setup_args.update(extra_args)
+
+    return dedent("""
+      from setuptools import setup
+
+      setup(**{setup_args!r})
+    """.format(setup_args=setup_args))
+
+  def with_foo_bar_ns_packages(content):
+    ns_packages = {
+      os.path.join(pkg, '__init__.py'): '__import__("pkg_resources").declare_namespace(__name__)'
+      for pkg in ('foo', 'foo/bar')
+    } if create_ns_packages else {}
+    ns_packages.update(content)
+    return ns_packages
+
+  content1 = with_foo_bar_ns_packages({
+    'foo/bar/spam.py': 'identify = lambda: 42',
+    'setup.py': create_foo_bar_setup('foo-bar-spam')
+  })
+
+  content2 = with_foo_bar_ns_packages({
+    'foo/bar/eggs.py': dedent("""
+      # NB: This only works when this content is unpacked loose on the filesystem!
+      def read_self():
+        with open(__file__) as fp:
+          return fp.read()
+    """)
+  })
+
+  content3 = with_foo_bar_ns_packages({
+    'foobaz': dedent("""\
+      #!python
+      import sys
+
+      from foo.bar import baz
+
+      sys.exit(baz.main())
+    """),
+    'foo/bar/baz.py': dedent("""
+      import sys
+
+      from foo.bar import eggs, spam
+
+      def main():
+        assert len(eggs.read_self()) > 0
+        return spam.identify()
+    """),
+    'setup.py': create_foo_bar_setup('foo-bar-baz', scripts=['foobaz'])
+  })
+
+  def add_requirements(builder, cache):
+    for resolved_dist in resolve(requirements, cache=cache, interpreter=builder.interpreter):
+      builder.add_requirement(resolved_dist.requirement)
+      builder.add_distribution(resolved_dist.distribution)
+
+  def add_wheel(builder, content):
+    with temporary_content(content) as project:
+      dist = WheelInstaller(project, interpreter=builder.interpreter).bdist()
+      builder.add_dist_location(dist)
+
+  def add_sources(builder, content):
+    with temporary_content(content) as project:
+      for path in content.keys():
+        builder.add_source(os.path.join(project, path), path)
+
+  interpreter = vendor.setup_interpreter(interpreter=interpreter)
+  with nested(temporary_dir(), temporary_dir()) as (root, cache):
+    pex_info1 = PexInfo.default()
+    pex_info1.zip_safe = False
+    pex1 = os.path.join(root, 'pex1.pex')
+    builder1 = PEXBuilder(interpreter=interpreter, pex_info=pex_info1)
+    add_requirements(builder1, cache)
+    add_wheel(builder1, content1)
+    add_sources(builder1, content2)
+    builder1.build(pex1)
+
+    pex_info2 = PexInfo.default()
+    pex_info2.pex_path = pex1
+    pex2 = os.path.join(root, 'pex2')
+    builder2 = PEXBuilder(path=pex2, interpreter=interpreter, pex_info=pex_info2)
+    add_requirements(builder2, cache)
+    add_wheel(builder2, content3)
+    builder2.set_script('foobaz')
+    builder2.freeze()
+
+    assert 42 == PEX(pex2, interpreter=interpreter).run(env=dict(PEX_VERBOSE='9'))
+
+
+@pytest.fixture
+def setuptools_requirement():
+  # We use a very old version of setuptools to prove the point the user version is what is used
+  # here and not the vendored version (when possible). A newer setuptools is needed though to work
+  # with python 3.
+  return 'setuptools==1.0' if PY2 else 'setuptools==17.0'
+
+
+def test_issues_598_explicit_any_interpreter(setuptools_requirement):
+  assert_force_local_implicit_ns_packages_issues_598(requirements=[setuptools_requirement],
+                                                     create_ns_packages=True)
+
+
+def test_issues_598_explicit_missing_requirement():
+  assert_force_local_implicit_ns_packages_issues_598(create_ns_packages=True)
+
+
+@pytest.fixture
+def python_35_interpreter():
+  # Python 3.5 supports implicit namespace packages.
+  return PythonInterpreter.from_binary(ensure_python_interpreter(PY35))
+
+
+def test_issues_598_implicit(python_35_interpreter):
+  assert_force_local_implicit_ns_packages_issues_598(interpreter=python_35_interpreter,
+                                                     create_ns_packages=False)
+
+
+def test_issues_598_implicit_explicit_mixed(python_35_interpreter, setuptools_requirement):
+  assert_force_local_implicit_ns_packages_issues_598(interpreter=python_35_interpreter,
+                                                     requirements=[setuptools_requirement],
+                                                     create_ns_packages=True)
+
+
 def normalize(path):
   return os.path.normpath(os.path.realpath(path)).lower()
 
diff --git a/tests/test_integration.py b/tests/test_integration.py
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -1196,3 +1196,13 @@ def test_setup_python_multiple_direct_markers():
       subprocess.check_call(py2_only_program, env=make_env(PATH=os.path.dirname(py36_interpreter)))
 
     subprocess.check_call(py2_only_program, env=make_env(PATH=os.path.dirname(py27_interpreter)))
+
+
+def test_force_local_implicit_ns_packages_issues_598():
+  # This was a minimal repro for the issue documented in #598.
+  with temporary_dir() as out:
+    tcl_pex = os.path.join(out, 'tcl.pex')
+    run_pex_command(['twitter.common.lang==0.3.9', '-o', tcl_pex])
+
+    subprocess.check_call([tcl_pex, '-c', 'from twitter.common.lang import Singleton'],
+                          env=make_env(PEX_FORCE_LOCAL='1', PEX_PATH=tcl_pex))