Repository: easybuilders/easybuild-framework

Issue:
find_software_name_for_patch can fail when non UTF8 files exist
Whilst trying
```
eb --update-pr 13453 Brotli-1.0.9_pc_link_flags.patch --pr-commit-msg="Add patch"
```
the code that tries to figure out where the patch belongs fails hard:
```
Traceback (most recent call last):
  File "/usr/lib64/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib64/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/apps/Common/software/EasyBuild/4.4.1/lib/python3.6/site-packages/easybuild/main.py", line 557, in <module>
    main()
  File "/apps/Common/software/EasyBuild/4.4.1/lib/python3.6/site-packages/easybuild/main.py", line 476, in main
    update_pr(options.update_pr, categorized_paths, ordered_ecs)
  File "/apps/Common/software/EasyBuild/4.4.1/lib/python3.6/site-packages/easybuild/tools/github.py", line 1876, in update_pr
    update_branch(branch_name, paths, ecs, github_account=github_account, commit_msg=commit_msg)
  File "/apps/Common/software/EasyBuild/4.4.1/lib/python3.6/site-packages/easybuild/tools/github.py", line 1848, in update_branch
    commit_msg=commit_msg)
  File "/apps/Common/software/EasyBuild/4.4.1/lib/python3.6/site-packages/easybuild/tools/github.py", line 861, in _easyconfigs_pr_common
    patch_specs = det_patch_specs(paths['patch_files'], file_info, [target_dir])
  File "/apps/Common/software/EasyBuild/4.4.1/lib/python3.6/site-packages/easybuild/tools/github.py", line 1051, in det_patch_specs
    soft_name = find_software_name_for_patch(patch_file, ec_dirs)
  File "/apps/Common/software/EasyBuild/4.4.1/lib/python3.6/site-packages/easybuild/tools/github.py", line 1079, in find_software_name_for_patch
    rawtxt = read_file(path)
  File "/apps/Common/software/EasyBuild/4.4.1/lib/python3.6/site-packages/easybuild/tools/filetools.py", line 209, in read_file
    txt = handle.read()
  File "/usr/lib64/python3.6/codecs.py", line 321, in decode
    (result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb4 in position 241240: invalid start byte
```

The file in question was
```
easybuild-easyconfigs/easybuild/easyconfigs/g/Grace/Grace-5.1.25-5build1.patch
```
which seems to be ISO-8859 encoded.

We have some options
1. I think this function should skip all patches, this just seems like a bug that it doesn't.
2. We could consider forcing UTF8 encoding? But it might be hard for patches..
3. We can just handle the errors here and proceed.
4. Make read_file encoding aware by some means?


Discussion:
https://github.com/easybuilders/easybuild-framework/blob/e74d2e23fb7b24c4bb6ebb457ef0e201427560c1/easybuild/tools/github.py#L1076
This condition should just look at `*.eb` files.

Resolving patch:
diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py
--- a/easybuild/tools/github.py
+++ b/easybuild/tools/github.py
@@ -839,7 +839,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
     # copy easyconfig files to right place
     target_dir = os.path.join(git_working_dir, pr_target_repo)
     print_msg("copying files to %s..." % target_dir)
-    file_info = COPY_FUNCTIONS[pr_target_repo](ec_paths, os.path.join(git_working_dir, pr_target_repo))
+    file_info = COPY_FUNCTIONS[pr_target_repo](ec_paths, target_dir)
 
     # figure out commit message to use
     if commit_msg:
@@ -1069,21 +1069,43 @@ def find_software_name_for_patch(patch_name, ec_dirs):
 
     soft_name = None
 
+    ignore_dirs = build_option('ignore_dirs')
     all_ecs = []
     for ec_dir in ec_dirs:
-        for (dirpath, _, filenames) in os.walk(ec_dir):
+        for (dirpath, dirnames, filenames) in os.walk(ec_dir):
+            # Exclude ignored dirs
+            if ignore_dirs:
+                dirnames[:] = [i for i in dirnames if i not in ignore_dirs]
             for fn in filenames:
-                if fn != 'TEMPLATE.eb' and not fn.endswith('.py'):
+                # TODO: In EasyBuild 5.x only check for '*.eb' files
+                if fn != 'TEMPLATE.eb' and os.path.splitext(fn)[1] not in ('.py', '.patch'):
                     path = os.path.join(dirpath, fn)
                     rawtxt = read_file(path)
                     if 'patches' in rawtxt:
                         all_ecs.append(path)
 
+    # Usual patch names are <software>-<version>_fix_foo.patch
+    # So search those ECs first
+    patch_stem = os.path.splitext(patch_name)[0]
+    # Extract possible sw name and version according to above scheme
+    # Those might be the same as the whole patch stem, which is OK
+    possible_sw_name = patch_stem.split('-')[0].lower()
+    possible_sw_name_version = patch_stem.split('_')[0].lower()
+
+    def ec_key(path):
+        filename = os.path.basename(path).lower()
+        # Put files with one of those as the prefix first, then sort by name
+        return (
+            not filename.startswith(possible_sw_name_version),
+            not filename.startswith(possible_sw_name),
+            filename
+        )
+    all_ecs.sort(key=ec_key)
+
     nr_of_ecs = len(all_ecs)
     for idx, path in enumerate(all_ecs):
         if soft_name:
             break
-        rawtxt = read_file(path)
         try:
             ecs = process_easyconfig(path, validate=False)
             for ec in ecs:


Test patch:
diff --git a/test/framework/github.py b/test/framework/github.py
--- a/test/framework/github.py
+++ b/test/framework/github.py
@@ -595,6 +595,16 @@ def test_github_find_patches(self):
         reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked')
         self.assertTrue(re.search(reg, txt))
 
+        self.assertEqual(gh.find_software_name_for_patch('test.patch', []), None)
+
+        # check behaviour of find_software_name_for_patch when non-UTF8 patch files are present (only with Python 3)
+        if sys.version_info[0] >= 3:
+            non_utf8_patch = os.path.join(self.test_prefix, 'problem.patch')
+            with open(non_utf8_patch, 'wb') as fp:
+                fp.write(bytes("+  ximage->byte_order=T1_byte_order; /* Set t1lib\xb4s byteorder */\n", 'iso_8859_1'))
+
+            self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None)
+
     def test_github_det_commit_status(self):
         """Test det_commit_status function."""