Repository: spack/spack

Issue:
spack install broken on python3
Using 1e18ace6472c94893cc567ca757032269e02460f and python3 I still get:
```
$ spack install pax-utils
==> Installing pax-utils
==> Fetching https://dev.gentoo.org/~vapier/dist/pax-utils-1.2.2.tar.xz
######################################################################## 100.0%
==> Staging archive: /Users/junghans/spack/var/spack/stage/pax-utils-1.2.2-xiaxjymvkqyzqrlkwwxr33nggcitgidv/pax-utils-1.2.2.tar.xz
==> Created stage in /Users/junghans/spack/var/spack/stage/pax-utils-1.2.2-xiaxjymvkqyzqrlkwwxr33nggcitgidv
==> Ran patch() for pax-utils
==> Building pax-utils [AutotoolsPackage]
==> Executing phase : 'autoreconf'
==> Executing phase : 'configure'
==> Executing phase : 'build'
==> Executing phase : 'install'
==> Error: UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf8 in position 40: invalid start byte
/Users/junghans/spack/lib/spack/spack/package.py:1291, in build_process:
     1219         def build_process(input_stream):
     1220             """Forked for each build. Has its own process and python
     1221                module space set up by build_environment.fork()."""
     1222
     1223             # We are in the child process. This means that our sys.stdin is
     1224             # equal to open(os.devnull). Python did this to prevent our process
     1225             # and the parent process from possible simultaneous reading from
     1226             # the original standard input. But we assume that the parent
     1227             # process is not going to read from it till we are done here,
     1228             # otherwise it should not have passed us the copy of the stream.
     1229             # Thus, we are free to work with the the copy (input_stream)
     1230             # however we want. For example, we might want to call functions
     1231             # (e.g. input()) that implicitly read from whatever stream is
     1232             # assigned to sys.stdin. Since we want them to work with the
     1233             # original input stream, we are making the following assignment:
     1234             sys.stdin = input_stream
     1235
     1236             start_time = time.time()
     1237             if not fake:
     1238                 if not skip_patch:
     1239                     self.do_patch()
     1240                 else:
     1241                     self.do_stage()
     1242
     1243             tty.msg(
     1244                 'Building {0} [{1}]'.format(self.name, self.build_system_class)
     1245             )
     1246
     1247             self.stage.keep = keep_stage
     1248             with self._stage_and_write_lock():
     1249                 # Run the pre-install hook in the child process after
     1250                 # the directory is created.
     1251                 spack.hooks.pre_install(self.spec)
     1252                 if fake:
     1253                     self.do_fake_install()
     1254                 else:
     1255                     # Do the real install in the source directory.
     1256                     self.stage.chdir_to_source()
     1257
     1258                     # Save the build environment in a file before building.
     1259                     env_path = join_path(os.getcwd(), 'spack-build.env')
     1260
     1261                     # Redirect I/O to a build log (and optionally to
     1262                     # the terminal)
     1263                     log_path = join_path(os.getcwd(), 'spack-build.out')
     1264
     1265                     # FIXME : refactor this assignment
     1266                     self.log_path = log_path
     1267                     self.env_path = env_path
     1268                     dump_environment(env_path)
     1269
     1270                     # Spawn a daemon that reads from a pipe and redirects
     1271                     # everything to log_path
     1272                     redirection_context = log_output(
     1273                         log_path,
     1274                         echo=verbose,
     1275                         force_color=sys.stdout.isatty(),
     1276                         debug=True,
     1277                         input_stream=input_stream
     1278                     )
     1279                     with redirection_context as log_redirection:
     1280                         for phase_name, phase in zip(
     1281                                 self.phases, self._InstallPhase_phases):
     1282                             tty.msg(
     1283                                 'Executing phase : \'{0}\''.format(phase_name)
     1284                             )
     1285                             # Redirect stdout and stderr to daemon pipe
     1286                             with log_redirection:
     1287                                 getattr(self, phase)(
     1288                                     self.spec, self.prefix)
     1289                     self.log()
     1290                 # Run post install hooks before build stage is removed.
  >> 1291                 spack.hooks.post_install(self.spec)
     1292
     1293             # Stop timer.
     1294             self._total_time = time.time() - start_time
     1295             build_time = self._total_time - self._fetch_time
     1296
     1297             tty.msg("Successfully installed %s" % self.name,
     1298                     "Fetch: %s.  Build: %s.  Total: %s." %
     1299                     (_hms(self._fetch_time), _hms(build_time),
     1300                      _hms(self._total_time)))
     1301             print_pkg(self.prefix)

See build log for details:
  /tmp/junghans/spack-stage/spack-stage-vsdhukt3/pax-utils-1.2.2/spack-build.out
```


Discussion:
Duplicate of #4004.

Resolving patch:
diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py
--- a/lib/spack/spack/hooks/sbang.py
+++ b/lib/spack/spack/hooks/sbang.py
@@ -38,9 +38,12 @@
 
 def shebang_too_long(path):
     """Detects whether a file has a shebang line that is too long."""
-    with open(path, 'r') as script:
+    if not os.path.isfile(path):
+        return False
+
+    with open(path, 'rb') as script:
         bytes = script.read(2)
-        if bytes != '#!':
+        if bytes != b'#!':
             return False
 
         line = bytes + script.readline()


Test patch:
diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py
--- a/lib/spack/spack/test/sbang.py
+++ b/lib/spack/spack/test/sbang.py
@@ -27,13 +27,16 @@
 """
 import os
 import stat
-import unittest
+import pytest
 import tempfile
 import shutil
 
 from llnl.util.filesystem import *
-from spack.hooks.sbang import filter_shebangs_in_directory
+
 import spack
+from spack.hooks.sbang import *
+from spack.util.executable import which
+
 
 short_line       = "#!/this/is/short/bin/bash\n"
 long_line        = "#!/this/" + ('x' * 200) + "/is/long\n"
@@ -43,14 +46,13 @@
 last_line        = "last!\n"
 
 
-class SbangTest(unittest.TestCase):
-
-    def setUp(self):
+class ScriptDirectory(object):
+    """Directory full of test scripts to run sbang instrumentation on."""
+    def __init__(self):
         self.tempdir = tempfile.mkdtemp()
 
-        # make sure we can ignore non-files
-        directory = os.path.join(self.tempdir, 'dir')
-        mkdirp(directory)
+        self.directory = os.path.join(self.tempdir, 'dir')
+        mkdirp(self.directory)
 
         # Script with short shebang
         self.short_shebang = os.path.join(self.tempdir, 'short')
@@ -71,48 +73,70 @@ def setUp(self):
             f.write(last_line)
 
         # Script already using sbang.
-        self.has_shebang = os.path.join(self.tempdir, 'shebang')
-        with open(self.has_shebang, 'w') as f:
+        self.has_sbang = os.path.join(self.tempdir, 'shebang')
+        with open(self.has_sbang, 'w') as f:
             f.write(sbang_line)
             f.write(long_line)
             f.write(last_line)
 
-    def tearDown(self):
+        # Fake binary file.
+        self.binary = os.path.join(self.tempdir, 'binary')
+        tar = which('tar', required=True)
+        tar('czf', self.binary, self.has_sbang)
+
+    def destroy(self):
         shutil.rmtree(self.tempdir, ignore_errors=True)
 
-    def test_shebang_handling(self):
-        filter_shebangs_in_directory(self.tempdir)
-
-        # Make sure this is untouched
-        with open(self.short_shebang, 'r') as f:
-            self.assertEqual(f.readline(), short_line)
-            self.assertEqual(f.readline(), last_line)
-
-        # Make sure this got patched.
-        with open(self.long_shebang, 'r') as f:
-            self.assertEqual(f.readline(), sbang_line)
-            self.assertEqual(f.readline(), long_line)
-            self.assertEqual(f.readline(), last_line)
-
-        # Make sure this got patched.
-        with open(self.lua_shebang, 'r') as f:
-            self.assertEqual(f.readline(), sbang_line)
-            self.assertEqual(f.readline(), lua_line_patched)
-            self.assertEqual(f.readline(), last_line)
-
-        # Make sure this is untouched
-        with open(self.has_shebang, 'r') as f:
-            self.assertEqual(f.readline(), sbang_line)
-            self.assertEqual(f.readline(), long_line)
-            self.assertEqual(f.readline(), last_line)
-
-    def test_shebang_handles_non_writable_files(self):
-        # make a file non-writable
-        st = os.stat(self.long_shebang)
-        not_writable_mode = st.st_mode & ~stat.S_IWRITE
-        os.chmod(self.long_shebang, not_writable_mode)
-
-        self.test_shebang_handling()
-
-        st = os.stat(self.long_shebang)
-        self.assertEqual(oct(not_writable_mode), oct(st.st_mode))
+
+@pytest.fixture
+def script_dir():
+    sdir = ScriptDirectory()
+    yield sdir
+    sdir.destroy()
+
+
+def test_shebang_handling(script_dir):
+    assert shebang_too_long(script_dir.lua_shebang)
+    assert shebang_too_long(script_dir.long_shebang)
+
+    assert not shebang_too_long(script_dir.short_shebang)
+    assert not shebang_too_long(script_dir.has_sbang)
+    assert not shebang_too_long(script_dir.binary)
+    assert not shebang_too_long(script_dir.directory)
+
+    filter_shebangs_in_directory(script_dir.tempdir)
+
+    # Make sure this is untouched
+    with open(script_dir.short_shebang, 'r') as f:
+        assert f.readline() == short_line
+        assert f.readline() == last_line
+
+    # Make sure this got patched.
+    with open(script_dir.long_shebang, 'r') as f:
+        assert f.readline() == sbang_line
+        assert f.readline() == long_line
+        assert f.readline() == last_line
+
+    # Make sure this got patched.
+    with open(script_dir.lua_shebang, 'r') as f:
+        assert f.readline() == sbang_line
+        assert f.readline() == lua_line_patched
+        assert f.readline() == last_line
+
+    # Make sure this is untouched
+    with open(script_dir.has_sbang, 'r') as f:
+        assert f.readline() == sbang_line
+        assert f.readline() == long_line
+        assert f.readline() == last_line
+
+
+def test_shebang_handles_non_writable_files(script_dir):
+    # make a file non-writable
+    st = os.stat(script_dir.long_shebang)
+    not_writable_mode = st.st_mode & ~stat.S_IWRITE
+    os.chmod(script_dir.long_shebang, not_writable_mode)
+
+    test_shebang_handling(script_dir)
+
+    st = os.stat(script_dir.long_shebang)
+    assert oct(not_writable_mode) == oct(st.st_mode)