Repository: pex-tool/pex

Issue:
--script not working with aws-cfn-bootstrap
I'm guessing this is NOT a `pex` issue, but I'm having a hard time figuring out what the issue is as I have been able to successfully use `--script` for other packages so maybe someone here can point me in the right direction. When running pex with the `aws-cfn-bootstrap`:

```
$ pex -v --script cfn-signal https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.zip
  aws-cfn-bootstrap 1.4solving distributions :: Packaging 
  pystache 0.5.4
  python-daemon 1.6.1
  lockfile 0.12.2
  setuptools 40.4.2
Traceback (most recent call last):
  File "/usr/local/bin/pex", line 11, in <module>
    sys.exit(main())
  File "/usr/local/lib/python2.7/dist-packages/pex/bin/pex.py", line 737, in main
    pex_builder = build_pex(reqs, options, resolver_options_builder)
  File "/usr/local/lib/python2.7/dist-packages/pex/bin/pex.py", line 680, in build_pex
    pex_builder.set_script(options.script)
  File "/usr/local/lib/python2.7/dist-packages/pex/pex_builder.py", line 225, in set_script
    script, ', '.join(str(d) for d in self._distributions)))
pex.pex_builder.InvalidExecutableSpecification: Could not find script 'cfn-signal' in any distribution python-daemon 1.6.1, setuptools 40.4.2, pystache 0.5.4, lockfile 0.12.2, aws-cfn-bootstrap 1.4 within PEX!
```
The `bin/cfn-signal` script is listed in the setup.py file. I have also tried the above pex command with `bin/cfn-signal` which shows the same error as above.

Inside a virtualenv, if I install this same zipfile, the `cfn-signal` is runnable as I expect.

Versions:
```
sherzberg@lla434d9ba5162~/workspace
$ pex --version
pex 1.4.6
sherzberg@lla434d9ba5162~/workspace
$ python --version
Python 2.7.12
```

Any ideas on what the issue could be?


Discussion:
This is definitely a bug! We do not properly normalize `-` to `_` in distribution names when searching for script paths. Fix coming.
Tangentially related: #383 That issue similarly concerned distributions with `-` in the name, but differently revolved around the generated wheel file name not properly using `_` in place of `-`; ie: the problem was on the wheel generation end and out of pex's control. In this case pex is failing to convert `-` to `_` when searching for a data directory internal to the wheel file itself; ie: the `-` to `_` conversion did occur during wheel generation but pex fails to take this into account when doing script searches.
Noting this is a dup of #443

Resolving patch:
diff --git a/pex/finders.py b/pex/finders.py
--- a/pex/finders.py
+++ b/pex/finders.py
@@ -16,6 +16,7 @@
 
 import os
 import pkgutil
+import re
 import sys
 import zipimport
 
@@ -99,12 +100,39 @@ def _remove_finder(importer, finder):
 class WheelMetadata(pkg_resources.EggMetadata):
   """Metadata provider for zipped wheels."""
 
+  @classmethod
+  def _escape(cls, filename_component):
+    # See: https://www.python.org/dev/peps/pep-0427/#escaping-and-unicode
+    return re.sub("[^\w\d.]+", "_", filename_component, re.UNICODE)
+
   @classmethod
   def _split_wheelname(cls, wheelname):
-    split_wheelname = wheelname.rsplit('-', 4)
-    assert len(split_wheelname) == 5, 'invalid wheel name: %s' % (wheelname)
-    split_wheelname[0] = split_wheelname[0].replace('-', '_')
-    return '-'.join(split_wheelname[:-3])
+    # See: https://www.python.org/dev/peps/pep-0427/#file-name-convention
+    assert wheelname.endswith('.whl'), 'invalid wheel name: %s' % wheelname
+    split_wheelname = wheelname.rsplit('-', 5)
+    assert len(split_wheelname) in (5, 6), 'invalid wheel name: %s' % wheelname
+    distribution, version = split_wheelname[:2]
+    return '%s-%s' % (distribution, version)
+
+  @classmethod
+  def data_dir(cls, wheel_path):
+    """Returns the internal path of the data dir for the given wheel.
+
+    As defined https://www.python.org/dev/peps/pep-0427/#the-data-directory
+
+    :rtype: str
+    """
+    return '%s.data' % cls._split_wheelname(os.path.basename(wheel_path))
+
+  @classmethod
+  def dist_info_dir(cls, wheel_path):
+    """Returns the internal path of the dist-info dir for the given wheel.
+
+    As defined here: https://www.python.org/dev/peps/pep-0427/#the-dist-info-directory
+
+    :rtype: str
+    """
+    return '%s.dist-info' % cls._split_wheelname(os.path.basename(wheel_path))
 
   def _setup_prefix(self):
     path = self.module_path
@@ -114,7 +142,7 @@ def _setup_prefix(self):
         self.egg_name = os.path.basename(path)
         # TODO(wickman) Test the regression where we have both upper and lower cased package
         # names.
-        self.egg_info = os.path.join(path, '%s.dist-info' % self._split_wheelname(self.egg_name))
+        self.egg_info = os.path.join(path, self.dist_info_dir(self.egg_name))
         self.egg_root = path
         break
       old = path
@@ -251,7 +279,7 @@ def get_script_from_whl(name, dist):
   # This can get called in different contexts; in some, it looks for files in the
   # wheel archives being used to produce a pex; in others, it looks for files in the
   # install wheel directory included in the pex. So we need to look at both locations.
-  datadir_name = "%s-%s.data" % (dist.project_name, dist.version)
+  datadir_name = WheelMetadata.data_dir(dist.location)
   wheel_scripts_dirs = ['bin', 'scripts',
                          os.path.join(datadir_name, "bin"),
                          os.path.join(datadir_name, "scripts")]
@@ -261,7 +289,7 @@ def get_script_from_whl(name, dist):
       # We always install wheel scripts into bin
       script_path = os.path.join(wheel_scripts_dir, name)
       return (
-          os.path.join(dist.egg_info, script_path),
+          os.path.join(dist.location, script_path),
           dist.get_resource_string('', script_path).replace(b'\r\n', b'\n').replace(b'\r', b'\n'))
   return None, None
 
@@ -300,15 +328,22 @@ def get_entrypoint(dist):
     script_entry = dist.get_entry_map().get('console_scripts', {}).get(script)
     if script_entry is not None:
       # Entry points are of the form 'foo = bar', we just want the 'bar' part.
-      return dist.key, str(script_entry).split('=')[1].strip()
+      return str(script_entry).split('=')[1].strip()
 
-  entries = frozenset(filter(None, (get_entrypoint(dist) for dist in dists)))
+  entries = {}
+  for dist in dists:
+    entry_point = get_entrypoint(dist)
+    if entry_point is not None:
+      entries[dist.key] = (dist, entry_point)
 
   if len(entries) > 1:
     raise RuntimeError(
         'Ambiguous script specification %s matches multiple entry points:\n\t%s' % (
-            script, '\n\t'.join('%s from %s' % (entry_point, key) for key, entry_point in entries)))
+            script,
+            '\n\t'.join('%r from %r' % (entry_point, dist)
+                        for dist, entry_point in entries.values())))
 
+  dist, entry_point = None, None
   if entries:
-    _, entry_point = next(iter(entries))
-    return entry_point
+    dist, entry_point = next(iter(entries.values()))
+  return dist, entry_point
diff --git a/pex/pex.py b/pex/pex.py
--- a/pex/pex.py
+++ b/pex/pex.py
@@ -463,14 +463,15 @@ def execute_interpreter(self):
   def execute_script(self, script_name):
     dists = list(self._activate())
 
-    entry_point = get_entry_point_from_console_script(script_name, dists)
+    dist, entry_point = get_entry_point_from_console_script(script_name, dists)
     if entry_point:
+      TRACER.log('Found console_script %r in %r' % (entry_point, dist))
       sys.exit(self.execute_entry(entry_point))
 
     dist, script_path, script_content = get_script_from_distributions(script_name, dists)
     if not dist:
-      raise self.NotFound('Could not find script %s in pex!' % script_name)
-    TRACER.log('Found script %s in %s' % (script_name, dist))
+      raise self.NotFound('Could not find script %r in pex!' % script_name)
+    TRACER.log('Found script %r in %r' % (script_name, dist))
     return self.execute_content(script_path, script_content, argv0=script_name)
 
   @classmethod
diff --git a/pex/pex_builder.py b/pex/pex_builder.py
--- a/pex/pex_builder.py
+++ b/pex/pex_builder.py
@@ -14,6 +14,7 @@
 from .finders import get_entry_point_from_console_script, get_script_from_distributions
 from .interpreter import PythonInterpreter
 from .pex_info import PexInfo
+from .tracer import TRACER
 from .util import CacheHelper, DistributionHelper
 
 BOOTSTRAP_ENVIRONMENT = b"""
@@ -207,17 +208,19 @@ def set_script(self, script):
     """
 
     # check if 'script' is a console_script
-    entry_point = get_entry_point_from_console_script(script, self._distributions)
+    dist, entry_point = get_entry_point_from_console_script(script, self._distributions)
     if entry_point:
       self.set_entry_point(entry_point)
+      TRACER.log('Set entrypoint to console_script %r in %r' % (entry_point, dist))
       return
 
     # check if 'script' is an ordinary script
-    script_path, _, _ = get_script_from_distributions(script, self._distributions)
-    if script_path:
+    dist, _, _ = get_script_from_distributions(script, self._distributions)
+    if dist:
       if self._pex_info.entry_point:
         raise self.InvalidExecutableSpecification('Cannot set both entry point and script of PEX!')
       self._pex_info.script = script
+      TRACER.log('Set entrypoint to script %r in %r' % (script, dist))
       return
 
     raise self.InvalidExecutableSpecification(


Test patch:
diff --git a/tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl b/tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl
new file mode 100644
Binary files /dev/null and b/tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl differ
diff --git a/tests/example_packages/eno-0.0.17-py2.7.egg b/tests/example_packages/eno-0.0.17-py2.7.egg
new file mode 100644
Binary files /dev/null and b/tests/example_packages/eno-0.0.17-py2.7.egg differ
diff --git a/tests/test_finders.py b/tests/test_finders.py
--- a/tests/test_finders.py
+++ b/tests/test_finders.py
@@ -1,15 +1,23 @@
 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import os
 import zipimport
 
 import pkg_resources
 import pytest
 
+from pex.compatibility import to_bytes
 from pex.finders import ChainedFinder
 from pex.finders import _add_finder as add_finder
 from pex.finders import _remove_finder as remove_finder
-from pex.finders import find_eggs_in_zip, get_entry_point_from_console_script, get_script_from_egg
+from pex.finders import (
+    find_eggs_in_zip,
+    find_wheels_in_zip,
+    get_entry_point_from_console_script,
+    get_script_from_egg,
+    get_script_from_whl
+)
 
 try:
   import mock
@@ -105,20 +113,46 @@ def test_remove_finder():
       mock_register_finder.assert_called_with('foo', pkg_resources.find_nothing)
 
 
+def test_get_script_from_egg_with_no_scripts():
+  # Make sure eggs without scripts don't cause errors.
+  egg_path = './tests/example_packages/Flask_Cache-0.13.1-py2.7.egg'
+  dists = list(find_eggs_in_zip(zipimport.zipimporter(egg_path), egg_path, only=True))
+  assert len(dists) == 1
+
+  dist = dists[0]
+  assert (None, None) == get_script_from_egg('non_existent_script', dist)
+
+
 def test_get_script_from_egg():
-  # Make sure eggs without scripts don't cause errors
-  dists = list(
-    find_eggs_in_zip(
-      zipimport.zipimporter('./tests/example_packages/Flask_Cache-0.13.1-py2.7.egg'),
-      './tests/example_packages/Flask_Cache-0.13.1-py2.7.egg',
-      only=True))
+  egg_path = './tests/example_packages/eno-0.0.17-py2.7.egg'
+  dists = list(find_eggs_in_zip(zipimport.zipimporter(egg_path), egg_path, only=True))
+  assert len(dists) == 1
+
+  dist = dists[0]
 
+  location, content = get_script_from_egg('run_eno_server', dist)
+  assert os.path.join(egg_path, 'EGG-INFO/scripts/run_eno_server') == location
+  assert content.startswith('#!'), 'Expected a `scripts` style script with shebang.'
+
+  assert (None, None) == get_script_from_egg('non_existent_script', dist)
+
+
+# In-part, tests a bug where the wheel distribution name has dashes as reported in:
+#   https://github.com/pantsbuild/pex/issues/443
+#   https://github.com/pantsbuild/pex/issues/551
+def test_get_script_from_whl():
+  whl_path = './tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl'
+  dists = list(find_wheels_in_zip(zipimport.zipimporter(whl_path), whl_path))
   assert len(dists) == 1
 
-  (location, content) = get_script_from_egg('non_existent_script', dists[0])
+  dist = dists[0]
+  assert 'aws-cfn-bootstrap' == dist.project_name
 
-  assert location is None
-  assert content is None
+  script_path, script_content = get_script_from_whl('cfn-signal', dist)
+  assert os.path.join(whl_path, 'aws_cfn_bootstrap-1.4.data/scripts/cfn-signal') == script_path
+  assert script_content.startswith(to_bytes('#!')), 'Expected a `scripts`-style script w/shebang.'
+
+  assert (None, None) == get_script_from_whl('non_existent_script', dist)
 
 
 class FakeDist(object):
@@ -134,7 +168,10 @@ def get_entry_map(self):
 def test_get_entry_point_from_console_script():
   dists = [FakeDist(key='fake', console_script_entry='bob= bob.main:run'),
            FakeDist(key='fake', console_script_entry='bob =bob.main:run')]
-  assert 'bob.main:run' == get_entry_point_from_console_script('bob', dists)
+
+  dist, entrypoint = get_entry_point_from_console_script('bob', dists)
+  assert 'bob.main:run' == entrypoint
+  assert dist in dists
 
 
 def test_get_entry_point_from_console_script_conflict():
@@ -147,4 +184,4 @@ def test_get_entry_point_from_console_script_conflict():
 def test_get_entry_point_from_console_script_dne():
   dists = [FakeDist(key='bob', console_script_entry='bob= bob.main:run'),
            FakeDist(key='fake', console_script_entry='bob =bob.main:run')]
-  assert None is get_entry_point_from_console_script('jane', dists)
+  assert (None, None) == get_entry_point_from_console_script('jane', dists)
diff --git a/tests/test_pex.py b/tests/test_pex.py
--- a/tests/test_pex.py
+++ b/tests/test_pex.py
@@ -211,7 +211,7 @@ def test_pex_script(installer_impl, project_name, zip_safe):
     env_copy['PEX_SCRIPT'] = 'hello_world'
     so, rc = run_simple_pex_test('', env=env_copy)
     assert rc == 1, so.decode('utf-8')
-    assert b'Could not find script hello_world' in so
+    assert b"Could not find script 'hello_world'" in so
 
     so, rc = run_simple_pex_test('', env=env_copy, dists=[bdist])
     assert rc == 0, so.decode('utf-8')