Repository: pex-tool/pex

Issue:
Pex erroneously warns about needing to use vendored `pkg_resources` for distributions with empty `namespace_packages.txt` metadata files.
As reported by @Eric-Arellano.

For example:
```
$ pex --version
pex 2.1.0
$ pex pycodestyle==2.5.0
/tmp/tmpwizaf8e_/.bootstrap/pex/environment.py:356: PEXWarning: The `pkg_resources` package was loaded from a pex vendored version when declaring namespace packages defined by odict_keys([pycodestyle 2.5.0 (/tmp/tmpwizaf8e_/.deps/pycodestyle-2.5.0-py2.py3-none-any.whl)]). These distributions should fix their `install_requires` to include `setuptools`
  pex_warnings.warn('The `pkg_resources` package was loaded from a pex vendored version when '
Python 3.8.1 (default, Dec 21 2019, 20:57:38) 
[GCC 9.2.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
(InteractiveConsole)
>>> 
```

But pycodestyle doesn't actually declare any namespace packages: https://github.com/PyCQA/pycodestyle/blob/2.5.0/setup.py#L34
```
$ curl -sSL https://files.pythonhosted.org/packages/0e/0c/04a353e104d2f324f8ee5f4b32012618c1c86dd79e52a433b64fceed511b/pycodestyle-2.5.0-py2.py3-none-any.whl -O
$ unzip -qc pycodestyle-2.5.0-py2.py3-none-any.whl pycodestyle-2.5.0.dist-info/namespace_packages.txt

$
```


Discussion:

Resolving patch:
diff --git a/pex/environment.py b/pex/environment.py
--- a/pex/environment.py
+++ b/pex/environment.py
@@ -312,7 +312,7 @@ def _resolve(self, working_set, reqs):
   @classmethod
   def _get_namespace_packages(cls, dist):
     if dist.has_metadata(cls._NAMESPACE_PACKAGE_METADATA_RESOURCE):
-      return dist.get_metadata_lines(cls._NAMESPACE_PACKAGE_METADATA_RESOURCE)
+      return list(dist.get_metadata_lines(cls._NAMESPACE_PACKAGE_METADATA_RESOURCE))
     else:
       return []
 
@@ -353,10 +353,17 @@ def declare_namespace_packages(cls, resolved_dists):
     # even then tear it all down when we hand off from the bootstrap to user code.
     pkg_resources, vendored = _import_pkg_resources()
     if vendored:
+      dists = '\n'.join(
+        '\n{index}. {dist} namespace packages:\n  {ns_packages}'.format(
+          index=index + 1,
+          dist=dist.as_requirement(),
+          ns_packages='\n  '.join(ns_packages)
+        ) for index, (dist, ns_packages) in enumerate(namespace_packages_by_dist.items())
+      )
       pex_warnings.warn('The `pkg_resources` package was loaded from a pex vendored version when '
-                        'declaring namespace packages defined by {dists}. These distributions '
+                        'declaring namespace packages defined by:\n{dists}\n\nThese distributions '
                         'should fix their `install_requires` to include `setuptools`'
-                        .format(dists=namespace_packages_by_dist.keys()))
+                        .format(dists=dists))
 
     for pkg in itertools.chain(*namespace_packages_by_dist.values()):
       if pkg in sys.modules:


Test patch:
diff --git a/tests/test_environment.py b/tests/test_environment.py
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -318,3 +318,33 @@ def test_activate_extras_issue_615():
       'Process failed with exit code {} and output:\n{}'.format(process.returncode, stderr)
     )
     assert to_bytes('{} 1.6.3'.format(os.path.basename(pb.path()))) == stdout.strip()
+
+
+def assert_namespace_packages_warning(distribution, version, expected_warning):
+  requirement = '{}=={}'.format(distribution, version)
+  pb = PEXBuilder()
+  for resolved_dist in resolver.resolve([requirement]):
+    pb.add_dist_location(resolved_dist.distribution.location)
+  pb.freeze()
+
+  process = PEX(pb.path()).run(args=['-c', ''], blocking=False, stderr=subprocess.PIPE)
+  _, stderr = process.communicate()
+  stderr_text = stderr.decode('utf8')
+
+  partial_warning_preamble = 'PEXWarning: The `pkg_resources` package was loaded'
+  partial_warning_detail = '{} namespace packages:'.format(requirement)
+
+  if expected_warning:
+    assert partial_warning_preamble in stderr_text
+    assert partial_warning_detail in stderr_text
+  else:
+    assert partial_warning_preamble not in stderr_text
+    assert partial_warning_detail not in stderr_text
+
+
+def test_present_non_empty_namespace_packages_metadata_does_warn():
+  assert_namespace_packages_warning('twitter.common.lang', '0.3.11', expected_warning=True)
+
+
+def test_present_but_empty_namespace_packages_metadata_does_not_warn():
+  assert_namespace_packages_warning('pycodestyle', '2.5.0', expected_warning=False)