Repository: easybuilders/easybuild-framework

Issue:
--from-pr does not (correctly) set the permissions
Use case: A pull request for an easyconfig contains an executable script which is to be run via `tests` which expects a list of commands or executables.

When trying this with `eb --from-pr` the executable permission from that file is lost making the test fail at the very end.

When checking the `*.patch` file from the PR I see `new file mode 100755`


Discussion:

Resolving patch:
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -2029,8 +2029,7 @@ def patch_step(self, beginpath=None):
             src = os.path.abspath(weld_paths(beginpath, srcpathsuffix))
             self.log.debug("Applying patch %s in path %s", patch, src)
 
-            if not apply_patch(patch['path'], src, copy=copy_patch, level=level):
-                raise EasyBuildError("Applying patch %s failed", patch['name'])
+            apply_patch(patch['path'], src, copy=copy_patch, level=level)
 
     def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
         """
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -1186,12 +1186,18 @@ def guess_patch_level(patched_files, parent_dir):
     return patch_level
 
 
-def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=False):
+def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=False, use_git=False):
     """
     Apply a patch to source code in directory dest
     - assume unified diff created with "diff -ru old new"
+
+    Raises EasyBuildError on any error and returns True on success
     """
 
+    if use_git_am:
+        _log.deprecated("'use_git_am' named argument in apply_patch function has been renamed to 'use_git'", '5.0')
+        use_git = True
+
     if build_option('extended_dry_run'):
         # skip checking of files in dry run mode
         patch_filename = os.path.basename(patch_file)
@@ -1218,61 +1224,59 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa
         raise EasyBuildError("Can't patch directory %s: no such directory", dest)
 
     # use absolute paths
-    apatch = os.path.abspath(patch_file)
-    adest = os.path.abspath(dest)
+    abs_patch_file = os.path.abspath(patch_file)
+    abs_dest = os.path.abspath(dest)
 
     # Attempt extracting the patch if it ends in .patch.gz, .patch.bz2, .patch.xz
-    # split in name + extension
-    apatch_root, apatch_file = os.path.split(apatch)
-    apatch_name, apatch_extension = os.path.splitext(apatch_file)
+    # split in stem (filename w/o extension) + extension
+    patch_stem, patch_extension = os.path.splitext(os.path.split(abs_patch_file)[1])
     # Supports only bz2, gz and xz. zip can be archives which are not supported.
-    if apatch_extension in ['.gz', '.bz2', '.xz']:
+    if patch_extension in ['.gz', '.bz2', '.xz']:
         # split again to get the second extension
-        apatch_subname, apatch_subextension = os.path.splitext(apatch_name)
-        if apatch_subextension == ".patch":
+        patch_subextension = os.path.splitext(patch_stem)[1]
+        if patch_subextension == ".patch":
             workdir = tempfile.mkdtemp(prefix='eb-patch-')
             _log.debug("Extracting the patch to: %s", workdir)
             # extracting the patch
-            apatch_dir = extract_file(apatch, workdir, change_into_dir=False)
-            change_dir(apatch_dir)
-            apatch = os.path.join(apatch_dir, apatch_name)
+            extracted_dir = extract_file(abs_patch_file, workdir, change_into_dir=False)
+            abs_patch_file = os.path.join(extracted_dir, patch_stem)
 
-    if level is None and build_option('extended_dry_run'):
-        level = '<derived>'
+    if use_git:
+        verbose = '--verbose ' if build_option('debug') else ''
+        patch_cmd = "git apply %s%s" % (verbose, abs_patch_file)
+    else:
+        if level is None and build_option('extended_dry_run'):
+            level = '<derived>'
 
-    elif level is None:
-        # guess value for -p (patch level)
-        # - based on +++ lines
-        # - first +++ line that matches an existing file determines guessed level
-        # - we will try to match that level from current directory
-        patched_files = det_patched_files(path=apatch)
+        elif level is None:
+            # guess value for -p (patch level)
+            # - based on +++ lines
+            # - first +++ line that matches an existing file determines guessed level
+            # - we will try to match that level from current directory
+            patched_files = det_patched_files(path=abs_patch_file)
 
-        if not patched_files:
-            raise EasyBuildError("Can't guess patchlevel from patch %s: no testfile line found in patch", apatch)
-            return
+            if not patched_files:
+                raise EasyBuildError("Can't guess patchlevel from patch %s: no testfile line found in patch",
+                                     abs_patch_file)
 
-        level = guess_patch_level(patched_files, adest)
+            level = guess_patch_level(patched_files, abs_dest)
 
-        if level is None:  # level can also be 0 (zero), so don't use "not level"
-            # no match
-            raise EasyBuildError("Can't determine patch level for patch %s from directory %s", patch_file, adest)
-        else:
-            _log.debug("Guessed patch level %d for patch %s" % (level, patch_file))
+            if level is None:  # level can also be 0 (zero), so don't use "not level"
+                # no match
+                raise EasyBuildError("Can't determine patch level for patch %s from directory %s", patch_file, abs_dest)
+            else:
+                _log.debug("Guessed patch level %d for patch %s" % (level, patch_file))
 
-    else:
-        _log.debug("Using specified patch level %d for patch %s" % (level, patch_file))
+        else:
+            _log.debug("Using specified patch level %d for patch %s" % (level, patch_file))
 
-    if use_git_am:
-        patch_cmd = "git am patch %s" % apatch
-    else:
-        patch_cmd = "patch -b -p%s -i %s" % (level, apatch)
+        patch_cmd = "patch -b -p%s -i %s" % (level, abs_patch_file)
 
-    out, ec = run.run_cmd(patch_cmd, simple=False, path=adest, log_ok=False, trace=False)
+    out, ec = run.run_cmd(patch_cmd, simple=False, path=abs_dest, log_ok=False, trace=False)
 
     if ec:
         raise EasyBuildError("Couldn't apply patch file %s. Process exited with code %s: %s", patch_file, ec, out)
-
-    return ec == 0
+    return True
 
 
 def apply_regex_substitutions(path, regex_subs, backup='.orig.eb'):
diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py
--- a/easybuild/tools/github.py
+++ b/easybuild/tools/github.py
@@ -454,7 +454,7 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None):
     elif not pr_closed:
         try:
             _log.debug("Trying to apply PR patch %s to %s...", diff_filepath, repo_target_branch)
-            apply_patch(diff_filepath, repo_target_branch, use_git_am=True)
+            apply_patch(diff_filepath, repo_target_branch, use_git=True)
             _log.info("Using %s which included PR patch to test PR #%s", repo_target_branch, pr)
             final_path = repo_target_branch
 


Test patch:
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -1856,7 +1856,7 @@ def run_checks():
         # single SHA256 checksum per source/patch: OK
         eb.cfg['checksums'] = [
             '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',  # toy-0.0.tar.gz
-            '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5',  # toy-*.patch
+            '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487',  # toy-*.patch
             '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458',  # toy-extra.txt]
         ]
         # no checksum issues
@@ -1865,7 +1865,7 @@ def run_checks():
         # SHA256 checksum with type specifier: OK
         eb.cfg['checksums'] = [
             ('sha256', '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'),  # toy-0.0.tar.gz
-            '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5',  # toy-*.patch
+            '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487',  # toy-*.patch
             ('sha256', '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458'),  # toy-extra.txt]
         ]
         # no checksum issues
@@ -1878,7 +1878,7 @@ def run_checks():
                 'a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd',
                 '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',
             ),
-            '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5',  # toy-*.patch
+            '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487',  # toy-*.patch
             '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458',  # toy-extra.txt
         ]
         # no checksum issues
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb
@@ -12,7 +12,7 @@ sources = [SOURCE_TAR_GZ]
 patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch']
 checksums = [
     ('adler32', '0x998410035'),
-    'e6785e1a721fc8bf79892e3ef41557c0',
+    'a99f2a72cee1689a2f7e3ace0356efb1',
 ]
 
 moduleclass = 'tools'
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1359,9 +1359,8 @@ def test_derive_alt_pypi_url(self):
     def test_apply_patch(self):
         """ Test apply_patch """
         testdir = os.path.dirname(os.path.abspath(__file__))
-        tmpdir = self.test_prefix
         toy_tar_gz = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz')
-        path = ft.extract_file(toy_tar_gz, tmpdir, change_into_dir=False)
+        path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False)
         toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch'
         toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn)
 
@@ -1403,6 +1402,49 @@ def test_apply_patch(self):
         ft.apply_patch(test_file, os.path.join(target_dir, 'subdir', 'target.txt'), copy=True)
         self.assertEqual(ft.read_file(os.path.join(target_dir, 'subdir', 'target.txt')), '123')
 
+        # cleanup and re-extract toy source tarball
+        ft.remove_dir(self.test_prefix)
+        ft.mkdir(self.test_prefix)
+        ft.change_dir(self.test_prefix)
+        path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False)
+
+        # test applying of patch with git
+        toy_source_path = os.path.join(self.test_prefix, 'toy-0.0', 'toy.source')
+        self.assertFalse("I'm a toy, and very proud of it" in ft.read_file(toy_source_path))
+
+        ft.apply_patch(toy_patch, self.test_prefix, use_git=True)
+        self.assertTrue("I'm a toy, and very proud of it" in ft.read_file(toy_source_path))
+
+        # construct patch that only adds a new file,
+        # this shouldn't break applying a patch with git even when no level is specified
+        new_file_patch = os.path.join(self.test_prefix, 'toy_new_file.patch')
+        new_file_patch_txt = '\n'.join([
+            "new file mode 100755",
+            "--- /dev/null\t1970-01-01 01:00:00.000000000 +0100",
+            "+++ b/toy-0.0/new_file.txt\t2020-08-18 12:31:57.000000000 +0200",
+            "@@ -0,0 +1 @@",
+            "+This is a new file\n",
+        ])
+        ft.write_file(new_file_patch, new_file_patch_txt)
+        ft.apply_patch(new_file_patch, self.test_prefix, use_git=True)
+        new_file_path = os.path.join(self.test_prefix, 'toy-0.0', 'new_file.txt')
+        self.assertEqual(ft.read_file(new_file_path), "This is a new file\n")
+
+        # cleanup & restore
+        ft.remove_dir(path)
+        path = ft.extract_file(toy_tar_gz, self.test_prefix, change_into_dir=False)
+
+        self.assertFalse("I'm a toy, and very proud of it" in ft.read_file(toy_source_path))
+
+        # mock stderr to catch deprecation warning caused by setting 'use_git_am'
+        self.allow_deprecated_behaviour()
+        self.mock_stderr(True)
+        ft.apply_patch(toy_patch, self.test_prefix, use_git_am=True)
+        stderr = self.get_stderr()
+        self.mock_stderr(False)
+        self.assertTrue("I'm a toy, and very proud of it" in ft.read_file(toy_source_path))
+        self.assertTrue("'use_git_am' named argument in apply_patch function has been renamed to 'use_git'" in stderr)
+
     def test_copy_file(self):
         """Test copy_file function."""
         testdir = os.path.dirname(os.path.abspath(__file__))
diff --git a/test/framework/options.py b/test/framework/options.py
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -4717,7 +4717,7 @@ def test_check_contrib_non_style(self):
             'checksums = [',
             "    None,  # toy-0.0.tar.gz",
             "    # toy-0.0_fix-silly-typo-in-printf-statement.patch",
-            "    '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5',",
+            "    '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487',",
             "    '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458',  # toy-extra.txt",
             ']\n',
         ])
@@ -4921,7 +4921,7 @@ def test_inject_checksums(self):
         stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True)
 
         toy_source_sha256 = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'
-        toy_patch_sha256 = '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5'
+        toy_patch_sha256 = '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'
         bar_tar_gz_sha256 = 'f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'
         bar_patch = 'bar-0.0_fix-silly-typo-in-printf-statement.patch'
         bar_patch_sha256 = '84db53592e882b5af077976257f9c7537ed971cb2059003fd4faa05d02cae0ab'
@@ -5056,7 +5056,7 @@ def test_inject_checksums(self):
             "^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+\.\.\.$",
             "^== injecting md5 checksums for sources & patches in test\.eb\.\.\.$",
             "^== \* toy-0.0\.tar\.gz: be662daa971a640e40be5c804d9d7d10$",
-            "^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: e6785e1a721fc8bf79892e3ef41557c0$",
+            r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: a99f2a72cee1689a2f7e3ace0356efb1$",
             "^== \* toy-extra\.txt: 3b0787b3bf36603ae1398c4a49097893$",
         ]
         for pattern in patterns:
@@ -5074,7 +5074,7 @@ def test_inject_checksums(self):
         ec = EasyConfigParser(test_ec).get_config_dict()
         checksums = [
             'be662daa971a640e40be5c804d9d7d10',
-            'e6785e1a721fc8bf79892e3ef41557c0',
+            'a99f2a72cee1689a2f7e3ace0356efb1',
             '3b0787b3bf36603ae1398c4a49097893',
         ]
         self.assertEqual(ec['checksums'], checksums)
diff --git a/test/framework/sandbox/sources/toy/toy-0.0_fix-silly-typo-in-printf-statement.patch b/test/framework/sandbox/sources/toy/toy-0.0_fix-silly-typo-in-printf-statement.patch
--- a/test/framework/sandbox/sources/toy/toy-0.0_fix-silly-typo-in-printf-statement.patch
+++ b/test/framework/sandbox/sources/toy/toy-0.0_fix-silly-typo-in-printf-statement.patch
@@ -1,8 +1,8 @@
---- a/toy-0.0/toy.source.orig	2013-11-30 14:43:48.000000000 +0100
-+++ b/toy-0.0/toy.source	2013-11-30 14:43:58.000000000 +0100
+--- a/toy-0.0.orig/toy.source	2014-03-06 18:48:16.000000000 +0100
++++ b/toy-0.0/toy.source	2020-08-18 12:19:35.000000000 +0200
 @@ -2,6 +2,6 @@
  
- int main(int argc, char* *argv[]){
+ int main(int argc, char* argv[]){
  
 -    printf("I'm a toy, and proud of it.\n");
 +    printf("I'm a toy, and very proud of it.\n");