Repository: easybuilders/easybuild-framework

Issue:
option to point framework to another software's patches would be useful
Example is my PoC for OpenMPI-CUDA which has the same patches as OpenMPI, duplicating them would be error prone. So something like
```
find_patches_in = ['OpenMPI']
```
could be useful.


Discussion:
Same issue for `MPItrampoline` which is internally building OpenMPI
Symlinks are not good enough?
How would you express (in an easyconfig file) that EasyBuild should go look for patch P in the directories it would consider for software S?
symlinks work, but are easy to miss.

in P.eb
```
find_patches_in = ['S']
```
(or fetch_patches_from) and taking a list of software names like my original example above.
And how does --new/update-pr handle symlinks?
Come to think of it, in my scenario (and here) what you really want is to find not just the patches, but also the sources. What you really want is to indicate additional locations which can also be used to find sources/patches
Indeed, the same goes for my case
Both patches and sources can be defined by a dict, we could introduce a new key that indicates the alternate place to search for them 
Ok, I took a look at https://github.com/easybuilders/easybuild-framework/blob/develop/easybuild/framework/easyblock.py#L689 and we could just add a kwarg there to use an alternate to `self.name` which can be provided as an entry in the sources/patches dict.

Resolving patch:
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -373,7 +373,8 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct
         :param checksum: checksum corresponding to source
         :param extension: flag if being called from collect_exts_file_info()
         """
-        filename, download_filename, extract_cmd, source_urls, git_config = None, None, None, None, None
+        filename, download_filename, extract_cmd = None, None, None
+        source_urls, git_config, alt_location = None, None, None
 
         if source is None:
             raise EasyBuildError("fetch_source called with empty 'source' argument")
@@ -387,6 +388,7 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct
             download_filename = source.pop('download_filename', None)
             source_urls = source.pop('source_urls', None)
             git_config = source.pop('git_config', None)
+            alt_location = source.pop('alt_location', None)
             if source:
                 raise EasyBuildError("Found one or more unexpected keys in 'sources' specification: %s", source)
 
@@ -401,7 +403,7 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
         path = self.obtain_file(filename, extension=extension, download_filename=download_filename,
                                 force_download=force_download, urls=source_urls, git_config=git_config,
-                                download_instructions=download_instructions)
+                                download_instructions=download_instructions, alt_location=alt_location)
         if path is None:
             raise EasyBuildError('No file found for source %s', filename)
 
@@ -468,7 +470,9 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
             patch_info['postinstall'] = patch_spec in post_install_patches
 
             force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES]
-            path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download)
+            alt_location = patch_info.pop('alt_location', None)
+            path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download,
+                                    alt_location=alt_location)
             if path:
                 self.log.debug('File %s found for patch %s', path, patch_spec)
                 patch_info['path'] = path
@@ -687,7 +691,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None, download_instructions=None):
+                    git_config=None, download_instructions=None, alt_location=None):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -698,11 +702,18 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         :param download_filename: filename with which the file should be downloaded, and then renamed to <filename>
         :param force_download: always try to download file, even if it's already available in source path
         :param git_config: dictionary to define how to download a git repository
+        :param download_instructions: instructions to manually add source (used for complex cases)
+        :param alt_location: alternative location to use instead of self.name
         """
         srcpaths = source_paths()
 
         update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
 
+        if alt_location is None:
+            location = self.name
+        else:
+            location = alt_location
+
         # should we download or just try and find it?
         if re.match(r"^(https?|ftp)://", filename):
             # URL detected, so let's try and download it
@@ -711,7 +722,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
             filename = url.split('/')[-1]
 
             # figure out where to download the file to
-            filepath = os.path.join(srcpaths[0], letter_dir_for(self.name), self.name)
+            filepath = os.path.join(srcpaths[0], letter_dir_for(location), location)
             if extension:
                 filepath = os.path.join(filepath, "extensions")
             self.log.info("Creating path %s to download file to" % filepath)
@@ -750,8 +761,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
             for path in ebpath + common_filepaths + srcpaths:
                 # create list of candidate filepaths
-                namepath = os.path.join(path, self.name)
-                letterpath = os.path.join(path, letter_dir_for(self.name), self.name)
+                namepath = os.path.join(path, location)
+                letterpath = os.path.join(path, letter_dir_for(location), location)
 
                 # most likely paths
                 candidate_filepaths = [
@@ -790,8 +801,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
                     break  # no need to try other source paths
 
-            name_letter = self.name.lower()[0]
-            targetdir = os.path.join(srcpaths[0], name_letter, self.name)
+            name_letter = location.lower()[0]
+            targetdir = os.path.join(srcpaths[0], name_letter, location)
 
             if foundfile:
                 if self.dry_run:
@@ -808,7 +819,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                 source_urls.extend(self.cfg['source_urls'])
 
                 # add https://sources.easybuild.io as fallback source URL
-                source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, self.name))
+                source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location))
 
                 mkdir(targetdir, parents=True)
 
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -1469,6 +1469,9 @@ def create_patch_info(patch_spec):
     """
     Create info dictionary from specified patch spec.
     """
+    # Valid keys that can be used in a patch spec dict
+    valid_keys = ['name', 'copy', 'level', 'sourcepath', 'alt_location']
+
     if isinstance(patch_spec, (list, tuple)):
         if not len(patch_spec) == 2:
             error_msg = "Unknown patch specification '%s', only 2-element lists/tuples are supported!"
@@ -1495,19 +1498,44 @@ def create_patch_info(patch_spec):
                                  str(patch_spec))
 
     elif isinstance(patch_spec, string_type):
-        allowed_patch_exts = ['.patch' + x for x in ('',) + ZIPPED_PATCH_EXTS]
-        if not any(patch_spec.endswith(x) for x in allowed_patch_exts):
-            msg = "Use of patch file with filename that doesn't end with correct extension: %s " % patch_spec
-            msg += "(should be any of: %s)" % (', '.join(allowed_patch_exts))
-            _log.deprecated(msg, '5.0')
+        validate_patch_spec(patch_spec)
         patch_info = {'name': patch_spec}
+    elif isinstance(patch_spec, dict):
+        patch_info = {}
+        for key in patch_spec.keys():
+            if key in valid_keys:
+                patch_info[key] = patch_spec[key]
+            else:
+                raise EasyBuildError("Wrong patch spec '%s', use of unknown key %s in dict (valid keys are %s)",
+                                     str(patch_spec), key, valid_keys)
+
+        # Dict must contain at least the patchfile name
+        if 'name' not in patch_info.keys():
+            raise EasyBuildError("Wrong patch spec '%s', when using a dict 'name' entry must be supplied",
+                                 str(patch_spec))
+        if 'copy' not in patch_info.keys():
+            validate_patch_spec(patch_info['name'])
+        else:
+            if 'sourcepath' in patch_info.keys() or 'level' in patch_info.keys():
+                raise EasyBuildError("Wrong patch spec '%s', you can't use 'sourcepath' or 'level' with 'copy' (since "
+                                     "this implies you want to copy a file to the 'copy' location)",
+                                     str(patch_spec))
     else:
-        error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s"
-        raise EasyBuildError(error_msg, patch_spec)
+        error_msg = "Wrong patch spec, should be string, 2-tuple with patch name + argument, or a dict " \
+                    "(with possible keys %s): %s" % (valid_keys, patch_spec)
+        raise EasyBuildError(error_msg)
 
     return patch_info
 
 
+def validate_patch_spec(patch_spec):
+    allowed_patch_exts = ['.patch' + x for x in ('',) + ZIPPED_PATCH_EXTS]
+    if not any(patch_spec.endswith(x) for x in allowed_patch_exts):
+        msg = "Use of patch file with filename that doesn't end with correct extension: %s " % patch_spec
+        msg += "(should be any of: %s)" % (', '.join(allowed_patch_exts))
+        _log.deprecated(msg, '5.0')
+
+
 def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=False, use_git=False):
     """
     Apply a patch to source code in directory dest


Test patch:
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -1619,11 +1619,12 @@ def test_fetch_patches(self):
             (toy_patch, 4),   # should be level 4
             (toy_patch, 'foobar'),  # sourcepath should be set to 'foobar'
             ('toy-0.0.tar.gz', 'some/path'),  # copy mode (not a .patch file)
+            {'name': toy_patch, 'level': 0, 'alt_location': 'alt_toy'},
         ]
         # check if patch levels are parsed correctly
         eb.fetch_patches(patch_specs=patches)
 
-        self.assertEqual(len(eb.patches), 4)
+        self.assertEqual(len(eb.patches), 5)
         self.assertEqual(eb.patches[0]['name'], toy_patch)
         self.assertEqual(eb.patches[0]['level'], 0)
         self.assertEqual(eb.patches[1]['name'], toy_patch)
@@ -1632,6 +1633,11 @@ def test_fetch_patches(self):
         self.assertEqual(eb.patches[2]['sourcepath'], 'foobar')
         self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz'),
         self.assertEqual(eb.patches[3]['copy'], 'some/path')
+        self.assertEqual(eb.patches[4]['name'], toy_patch)
+        self.assertEqual(eb.patches[4]['level'], 0)
+        testdir = os.path.abspath(os.path.dirname(__file__))
+        sandbox_sources = os.path.join(testdir, 'sandbox', 'sources')
+        self.assertEqual(eb.patches[4]['path'], os.path.join(sandbox_sources, 'alt_toy', toy_patch))
 
         patches = [
             ('toy-0.0_level4.patch', False),  # should throw an error, only int's an strings allowed here
@@ -1644,6 +1650,7 @@ def test_obtain_file(self):
         testdir = os.path.abspath(os.path.dirname(__file__))
         sandbox_sources = os.path.join(testdir, 'sandbox', 'sources')
         toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball)
+        alt_toy_tarball_path = os.path.join(sandbox_sources, 'alt_toy', toy_tarball)
         tmpdir = tempfile.mkdtemp()
         tmpdir_subdir = os.path.join(tmpdir, 'testing')
         mkdir(tmpdir_subdir, parents=True)
@@ -1658,20 +1665,31 @@ def test_obtain_file(self):
         res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
         self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball))
 
+        # 'downloading' a file to (first) alternative sourcepath works
+        res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir], alt_location='alt_toy')
+        self.assertEqual(res, os.path.join(tmpdir, 'a', 'alt_toy', toy_tarball))
+
         # finding a file in sourcepath works
         init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)])
         res = eb.obtain_file(toy_tarball)
         self.assertEqual(res, toy_tarball_path)
 
+        # finding a file in the alternate location works
+        res = eb.obtain_file(toy_tarball, alt_location='alt_toy')
+        self.assertEqual(res, alt_toy_tarball_path)
+
         # sourcepath has preference over downloading
         res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
         self.assertEqual(res, toy_tarball_path)
+        res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir], alt_location='alt_toy')
+        self.assertEqual(res, alt_toy_tarball_path)
 
         init_config(args=["--sourcepath=%s:%s" % (tmpdir, sandbox_sources)])
 
         # clean up toy tarballs in tmpdir, so the one in sourcepath is found
         remove_file(os.path.join(tmpdir, toy_tarball))
         remove_file(os.path.join(tmpdir, 't', 'toy', toy_tarball))
+        remove_file(os.path.join(tmpdir, 'a', 'alt_toy', toy_tarball))
 
         # enabling force_download results in re-downloading, even if file is already in sourcepath
         self.mock_stderr(True)
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1631,6 +1631,15 @@ def test_create_patch_info(self):
         self.assertEqual(ft.create_patch_info(('foo.patch', 1)), {'name': 'foo.patch', 'level': 1})
         self.assertEqual(ft.create_patch_info(('foo.patch', 'subdir')), {'name': 'foo.patch', 'sourcepath': 'subdir'})
         self.assertEqual(ft.create_patch_info(('foo.txt', 'subdir')), {'name': 'foo.txt', 'copy': 'subdir'})
+        self.assertEqual(ft.create_patch_info({'name': 'foo.patch'}), {'name': 'foo.patch'})
+        self.assertEqual(ft.create_patch_info({'name': 'foo.patch', 'sourcepath': 'subdir'}),
+                         {'name': 'foo.patch', 'sourcepath': 'subdir'})
+        self.assertEqual(ft.create_patch_info({'name': 'foo.txt', 'copy': 'subdir'}),
+                         {'name': 'foo.txt', 'copy': 'subdir'})
+        self.assertEqual(ft.create_patch_info({'name': 'foo.patch', 'sourcepath': 'subdir', 'alt_location': 'alt'}),
+                         {'name': 'foo.patch', 'sourcepath': 'subdir', 'alt_location': 'alt'})
+        self.assertEqual(ft.create_patch_info({'name': 'foo.txt', 'copy': 'subdir', 'alt_location': 'alt'}),
+                         {'name': 'foo.txt', 'copy': 'subdir', 'alt_location': 'alt'})
 
         self.allow_deprecated_behaviour()
         self.mock_stderr(True)
@@ -1650,7 +1659,12 @@ def test_create_patch_info(self):
         # faulty input
         error_msg = "Wrong patch spec"
         self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, None)
-        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, {'name': 'foo.patch'})
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, {'copy': 'subdir'})
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, {'name': 'foo.txt', 'random': 'key'})
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info,
+                              {'name': 'foo.txt', 'copy': 'subdir', 'sourcepath': 'subdir'})
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info,
+                              {'name': 'foo.txt', 'copy': 'subdir', 'level': 1})
         self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, ('foo.patch', [1, 2]))
         error_msg = "Unknown patch specification"
         self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, ('foo.patch', 1, 'subdir'))
diff --git a/test/framework/sandbox/sources/alt_toy b/test/framework/sandbox/sources/alt_toy
new file mode 120000
--- /dev/null
+++ b/test/framework/sandbox/sources/alt_toy
@@ -0,0 +1 @@
+toy
\ No newline at end of file