Repository: easybuilders/easybuild-framework

Issue:
fix_python_shebang incorrectly adds shebang to binary files when running eb on top of Python 2
This was introduced in #3183

Example: `GLib-2.62.0-GCCcore-8.3.0.eb` (which uses `fix_python_shebang_for = ['bin/*']`).

`bin/glib-compile-schemas` gets `#!/usr/bin/env python` added at the start, because no shebang was found at all, but that's horribly wrong since `glib-compile-schemas` is a binary...

This only happens when running `eb` on top of Python 2, since with Python 3 the `try``/except` mechanism around `read_file` in the `fix_shebang` method will ensure we're not trying to patch a binary file.
This doesn't work in Python 2 though, since there `read_file` has no problems with reading a binary file as if it were text...


Discussion:

Resolving patch:
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -75,7 +75,7 @@
 from easybuild.tools.filetools import change_dir, convert_name, compute_checksum, copy_file, derive_alt_pypi_url
 from easybuild.tools.filetools import diff_files, download_file, encode_class_name, extract_file
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
-from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
+from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file, dir_contains_files
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
@@ -2223,20 +2223,24 @@ def fix_shebang(self):
                                   lang, shebang, glob_pattern, paths)
                     for path in paths:
                         # check whether file should be patched by checking whether it has a shebang we want to tweak;
-                        # this also helps to skip binary files we may be hitting
+                        # this also helps to skip binary files we may be hitting (but only with Python 3)
                         try:
                             contents = read_file(path, mode='r')
                             should_patch = shebang_regex.match(contents)
                         except (TypeError, UnicodeDecodeError):
                             should_patch = False
+                            contents = None
 
+                        # if an existing shebang is found, patch it
                         if should_patch:
                             contents = shebang_regex.sub(shebang, contents)
                             write_file(path, contents)
-                        elif not contents.startswith('#!'):
+
+                        # if no shebang is present at all, add one (but only for non-binary files!)
+                        elif contents is not None and not is_binary(contents) and not contents.startswith('#!'):
                             self.log.info("The file '%s' doesn't have any shebang present, inserting it as first line.",
                                           path)
-                            contents = shebang + "\n" + contents
+                            contents = shebang + '\n' + contents
                             write_file(path, contents)
 
     def post_install_step(self):
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -241,6 +241,13 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
         raise EasyBuildError("Failed to write to %s: %s", path, err)
 
 
+def is_binary(contents):
+    """
+    Check whether given bytestring represents the contents of a binary file or not.
+    """
+    return isinstance(contents, bytes) and b'\00' in bytes(contents)
+
+
 def resolve_path(path):
     """
     Return fully resolved path for given path.


Test patch:
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -656,6 +656,16 @@ def test_read_write_file(self):
         # test use of 'mode' in read_file
         self.assertEqual(ft.read_file(foo, mode='rb'), b'bar')
 
+    def test_is_binary(self):
+        """Test is_binary function."""
+
+        for test in ['foo', '', b'foo', b'', "This is just a test", b"This is just a test", b"\xa0"]:
+            self.assertFalse(ft.is_binary(test))
+
+        self.assertTrue(ft.is_binary(b'\00'))
+        self.assertTrue(ft.is_binary(b"File is binary when it includes \00 somewhere"))
+        self.assertTrue(ft.is_binary(ft.read_file('/bin/ls', mode='rb')))
+
     def test_det_patched_files(self):
         """Test det_patched_files function."""
         toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch'
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -2377,6 +2377,10 @@ def test_fix_shebang(self):
         test_ec_txt = '\n'.join([
             toy_ec_txt,
             "postinstallcmds = ["
+            # copy of bin/toy to use in fix_python_shebang_for and fix_perl_shebang_for
+            "    'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.python',",
+            "    'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.perl',",
+
             # hardcoded path to bin/python
             "   'echo \"#!/usr/bin/python\\n# test\" > %(installdir)s/bin/t1.py',",
             # hardcoded path to bin/python3.6
@@ -2413,14 +2417,23 @@ def test_fix_shebang(self):
             "   'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b2.sh',",
 
             "]",
-            "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy', 'bin/b1.sh']",
-            "fix_perl_shebang_for = ['bin/*.pl', 'bin/b2.sh']",
+            "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']",
+            "fix_perl_shebang_for = ['bin/*.pl', 'bin/b2.sh', 'bin/toy.perl']",
         ])
         write_file(test_ec, test_ec_txt)
         self.test_toy_build(ec_file=test_ec, raise_error=True)
 
         toy_bindir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin')
 
+        # bin/toy and bin/toy2 should *not* be patched, since they're binary files
+        toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb')
+        for fn in ['toy.perl', 'toy.python']:
+            fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb')
+            # no shebang added
+            self.assertFalse(fn_txt.startswith(b"#!/"))
+            # exact same file as original binary (untouched)
+            self.assertEqual(toy_txt, fn_txt)
+
         # no re.M, this should match at start of file!
         py_shebang_regex = re.compile(r'^#!/usr/bin/env python\n# test$')
         for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: