Repository: easybuilders/easybuild-framework

Issue:
--from-pr doesn't merge easyconfigs with develop first
This occured in https://github.com/hpcugent/easybuild-easyconfigs/pull/1016, where the branch was outdated, and the easyconfig didn't include `easyblock` yet.

This is rather worrying...



Discussion:
Change to make:
- download latest develop tarball for easybuild-easyconfigs
- apply PR patch
- use patched easyconfigs from there

This will also fix the problem with not finding patch files that are already in develop.

@Caylo it would be great to resolve this in the next couple of days, so we can include the fix for this issue in the upcoming EasyBuild v2.8.2

Can you please take a look at this?

All the necessary functionality is already there I think, it's just putting the pieces together...

fixed with #1832, thanks @Caylo!

reopening since #1832 was reverted in #1835, because it introduced a regression...

We'll need to be more careful with this, especially w.r.t. PRs that are already merged.

Maybe we should emit a clear warning that the patch could not be applied on top of `develop`, and just continue with the 'raw' easyconfigs straight from the PR (if `--force` or something similar is enabled), i.e. fall back like it was before #1832?

Maybe this is a good option: if `eb --from-pr 1234` fails because the patch can't be merged for whatever reason, print a clear warning message mentioning that, and support using something like `eb --from-pr 1234*` which indicates that `eb` should continue even if the patch could not be applied on `develop` (relying on `--force` for the latter is not a good option since that implies reinstalling software that is already there, you don't want that as a side-effect).


Resolving patch:
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -718,11 +718,12 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None):
         _log.debug("Using specified patch level %d for patch %s" % (level, patch_file))
 
     patch_cmd = "patch -b -p%s -i %s" % (level, apatch)
-    result = run.run_cmd(patch_cmd, simple=True, path=adest)
-    if not result:
-        raise EasyBuildError("Patching with patch %s failed", patch_file)
+    out, ec = run.run_cmd(patch_cmd, simple=False, path=adest, log_ok=False)
 
-    return result
+    if ec:
+        raise EasyBuildError("Couldn't apply patch file %s. Process exited with code %s: %s", patch_file, ec, out)
+
+    return ec == 0
 
 
 def apply_regex_substitutions(path, regex_subs):
diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py
--- a/easybuild/tools/github.py
+++ b/easybuild/tools/github.py
@@ -47,8 +47,8 @@
 from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option
-from easybuild.tools.filetools import det_patched_files, download_file, extract_file, mkdir, read_file
-from easybuild.tools.filetools import which, write_file
+from easybuild.tools.filetools import apply_patch, det_patched_files, download_file, extract_file
+from easybuild.tools.filetools import mkdir, read_file, which, write_file
 from easybuild.tools.systemtools import UNKNOWN, get_tool_version
 from easybuild.tools.utilities import only_if_module_is_available
 
@@ -85,6 +85,7 @@
 GITHUB_FILE_TYPE = u'file'
 GITHUB_MAX_PER_PAGE = 100
 GITHUB_MERGEABLE_STATE_CLEAN = 'clean'
+GITHUB_PR = 'pull'
 GITHUB_RAW = 'https://raw.githubusercontent.com'
 GITHUB_STATE_CLOSED = 'closed'
 HTTP_STATUS_OK = 200
@@ -301,6 +302,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_
     _log.debug("%s downloaded to %s, extracting now" % (base_name, path))
 
     extracted_path = os.path.join(extract_file(target_path, path), extracted_dir_name)
+
     # check if extracted_path exists
     if not os.path.isdir(extracted_path):
         raise EasyBuildError("%s should exist and contain the repo %s at branch %s", extracted_path, repo, branch)
@@ -313,6 +315,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_
 
 def fetch_easyconfigs_from_pr(pr, path=None, github_user=None):
     """Fetch patched easyconfig files for a particular PR."""
+
     if github_user is None:
         github_user = build_option('github_user')
     if path is None:
@@ -332,25 +335,31 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None):
         raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)",
                              pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, status, pr_data)
 
-    # 'clean' on successful (or missing) test, 'unstable' on failed tests
+    # if PR is open and mergable, download develop and patch
     stable = pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN
+    closed = pr_data['state'] == GITHUB_STATE_CLOSED and not pr_data['merged']
+
+    # 'clean' on successful (or missing) test, 'unstable' on failed tests or merge conflict
     if not stable:
         _log.warning("Mergeable state for PR #%d is not '%s': %s.",
                      pr, GITHUB_MERGEABLE_STATE_CLEAN, pr_data['mergeable_state'])
 
-    for key, val in sorted(pr_data.items()):
-        _log.debug("\n%s:\n\n%s\n" % (key, val))
+    if (stable or pr_data['merged']) and not closed:
+        # whether merged or not, download develop
+        path = download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='develop', path=path)
 
     # determine list of changed files via diff
     diff_fn = os.path.basename(pr_data['diff_url'])
     diff_filepath = os.path.join(path, diff_fn)
     download_file(diff_fn, pr_data['diff_url'], diff_filepath, forced=True)
     diff_txt = read_file(diff_filepath)
-    os.remove(diff_filepath)
 
     patched_files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True)
     _log.debug("List of patched files: %s" % patched_files)
 
+    for key, val in sorted(pr_data.items()):
+        _log.debug("\n%s:\n\n%s\n" % (key, val))
+
     # obtain last commit
     # get all commits, increase to (max of) 100 per page
     if pr_data['commits'] > GITHUB_MAX_PER_PAGE:
@@ -360,30 +369,34 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None):
     last_commit = commits_data[-1]
     _log.debug("Commits: %s, last commit: %s" % (commits_data, last_commit['sha']))
 
-    # obtain most recent version of patched files
-    for patched_file in patched_files:
-        # path to patch file, incl. subdir it is in
-        fn = os.path.sep.join(patched_file.split(os.path.sep)[-2:])
-        sha = last_commit['sha']
-        full_url = URL_SEPARATOR.join([GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, sha, patched_file])
-        _log.info("Downloading %s from %s" % (fn, full_url))
-        download_file(fn, full_url, path=os.path.join(path, fn), forced=True)
-
-    # sanity check: make sure all patched files are downloaded
-    all_files = [os.path.sep.join(f.split(os.path.sep)[-2:]) for f in patched_files]
-
-    tmp_files = []
-    for (dirpath, _, filenames) in os.walk(path):
-        tmp_files.extend([os.path.join(os.path.basename(dirpath), f) for f in filenames])
+    if not(pr_data['merged']):
+        if not stable or closed:
+            print "\n*** WARNING: Using easyconfigs from unstable/closed PR #%s ***\n" % pr
+            # obtain most recent version of patched files
+            for patched_file in patched_files:
+                # path to patch file, incl. subdir it is in
+                fn = os.path.sep.join(patched_file.split(os.path.sep)[-2:])
+                sha = last_commit['sha']
+                full_url = URL_SEPARATOR.join([GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, sha, patched_file])
+                _log.info("Downloading %s from %s" % (fn, full_url))
+                download_file(fn, full_url, path=os.path.join(path, patched_file), forced=True)
+        else:
+            apply_patch(diff_filepath, path, level=1)
 
-    if not sorted(tmp_files) == sorted(all_files):
-        raise EasyBuildError("Not all patched files were downloaded to %s: %s vs %s", path, tmp_files, all_files)
+    os.remove(diff_filepath)
 
-    ec_files = [os.path.join(path, f) for f in tmp_files]
+    # sanity check: make sure all patched files are downloaded
+    ec_files = []
+    for patched in patched_files:
+        if os.path.exists(os.path.join(path, patched)):
+            ec_files.append(os.path.join(path, patched))
+        else:
+            raise EasyBuildError("Couldn't find path to patched file %s", os.path.join(path, patched))
 
     return ec_files
 
 
+
 def create_gist(txt, fn, descr=None, github_user=None):
     """Create a gist with the provided text."""
     if descr is None:


Test patch:
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -694,6 +694,21 @@ def test_derive_alt_pypi_url(self):
         url = 'https://pypi.python.org/packages/source/n/nosuchpackageonpypiever/nosuchpackageonpypiever-0.0.0.tar.gz'
         self.assertEqual(ft.derive_alt_pypi_url(url), None)
 
+    def test_apply_patch(self):
+        """ Test apply_patch """
+        testdir = os.path.dirname(os.path.abspath(__file__))
+        tmpdir = self.test_prefix
+        path = ft.extract_file(os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz'), tmpdir)
+        toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0_typo.patch')
+
+        self.assertTrue(ft.apply_patch(toy_patch, path))
+        patched = ft.read_file(os.path.join(path, 'toy-0.0', 'toy.source'))
+        pattern = "I'm a toy, and very proud of it"
+        self.assertTrue(pattern in patched)
+
+        # trying the patch again should fail
+        self.assertErrorRegex(EasyBuildError, "Couldn't apply patch file", ft.apply_patch, toy_patch, path)
+
 
 def suite():
     """ returns all the testcases in this module """
diff --git a/test/framework/github.py b/test/framework/github.py
--- a/test/framework/github.py
+++ b/test/framework/github.py
@@ -132,7 +132,12 @@ def test_fetch_easyconfigs_from_pr(self):
         try:
             ec_files = gh.fetch_easyconfigs_from_pr(2481, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
             self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files]))
-            self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in glob.glob(os.path.join(tmpdir, '*', '*'))]))
+
+            tmp_files = []
+            for (dirpath, _, filenames) in os.walk(tmpdir):
+                tmp_files.extend([f for f in filenames])
+
+            self.assertTrue(all(ecf in tmp_files for ecf in [os.path.basename(f) for f in ec_files]))
 
             # PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed)
             err_msg = "PR #897 contains more than .* commits, can't obtain last commit"