Repository: easybuilders/easybuild-easyconfigs

Issue:
Incorrect MASS 7.3-51.4 checksum value of R-3.6.0-foss-2019a.eb
can't build R-3.6.0-foss-2019a.eb
```
    ('MASS', '7.3-51.4', {
        'checksums': 
              ['9911d546a8d29dc906b46cb53ef8aad76d23566f4fc3b52778a1201f8a9b2c74'],
        }),
```
this -> 9911d546a8d29dc906b46cb53ef8aad76d23566f4fc3b52778a1201f8a9b2c74
should be -> 844270a2541eaed420871dfb61d681aa67ee57126645fb6b144b436c25698eeb


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,5 @@
 ##
-# Copyright 2012-2019 Ghent University
+# Copyright 2012-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -44,7 +44,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = '4.1.0'
+VERSION = '4.1.1'
 
 MAJ_VER = VERSION.split('.')[0]
 MAJMIN_VER = '.'.join(VERSION.split('.')[0:2])


Test patch:
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -30,7 +30,10 @@ jobs:
 
     - name: install OS & Python packages
       run: |
-        sudo apt-get update
+        # apt-get update is disabled for now, because it was failing in GitHub since Fri Dec 13th 2019,
+        # due to 'Conflicting distribution' issue
+        # sudo apt-get update
+        #
         # for modules tool
         sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
diff --git a/test/__init__.py b/test/__init__.py
--- a/test/__init__.py
+++ b/test/__init__.py
@@ -1,5 +1,5 @@
 ##
-# Copyright 2009-2019 Ghent University
+# Copyright 2009-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/easyconfigs/easyconfigs.py b/test/easyconfigs/easyconfigs.py
--- a/test/easyconfigs/easyconfigs.py
+++ b/test/easyconfigs/easyconfigs.py
@@ -1,5 +1,5 @@
 ##
-# Copyright 2013-2019 Ghent University
+# Copyright 2013-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -182,14 +182,16 @@ def check_dep_vars(self, dep, dep_vars):
             if len(empty_vsuff_vars) == 1:
                 dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != empty_vsuff_vars[0])
 
-        # multiple variants of HTSlib is OK as long as they are deps for a matching version of BCFtools
-        if dep == 'HTSlib' and len(dep_vars) > 1:
-            for key in list(dep_vars):
-                ecs = dep_vars[key]
-                # filter out HTSlib variants that are only used as dependency for BCFtools with same version
-                htslib_ver = re.search('^version: (?P<ver>[^;]+);', key).group('ver')
-                if all(ec.startswith('BCFtools-%s-' % htslib_ver) for ec in ecs):
-                    dep_vars.pop(key)
+        # multiple variants of HTSlib is OK as long as they are deps for a matching version of BCFtools;
+        # same goes for WRF and WPS
+        for dep_name, parent_name in [('HTSlib', 'BCFtools'), ('WRF', 'WPS')]:
+            if dep == dep_name and len(dep_vars) > 1:
+                for key in list(dep_vars):
+                    ecs = dep_vars[key]
+                    # filter out dep variants that are only used as dependency for parent with same version
+                    dep_ver = re.search('^version: (?P<ver>[^;]+);', key).group('ver')
+                    if all(ec.startswith('%s-%s-' % (parent_name, dep_ver)) for ec in ecs) and len(dep_vars) > 1:
+                        dep_vars.pop(key)
 
         # multiple versions of Boost is OK as long as they are deps for a matching Boost.Python
         if dep == 'Boost' and len(dep_vars) > 1:
@@ -208,7 +210,7 @@ def check_dep_vars(self, dep, dep_vars):
 
         # for some dependencies, we allow exceptions for software that depends on a particular version,
         # as long as that's indicated by the versionsuffix
-        if dep in ['ASE', 'Boost', 'Java', 'Lua', 'PLUMED', 'R'] and len(dep_vars) > 1:
+        if dep in ['ASE', 'Boost', 'Java', 'Lua', 'PLUMED', 'R', 'TensorFlow'] and len(dep_vars) > 1:
             for key in list(dep_vars):
                 dep_ver = re.search('^version: (?P<ver>[^;]+);', key).group('ver')
                 # use version of Java wrapper rather than full Java version
@@ -239,9 +241,9 @@ def check_dep_vars(self, dep, dep_vars):
 
         # some software packages require an old version of a particular dependency
         old_dep_versions = {
-            # libxc 2.x or 3.x is required by ABINIT, AtomPAW, CP2K, GPAW, PySCF, WIEN2k
+            # libxc 2.x or 3.x is required by ABINIT, AtomPAW, CP2K, GPAW, horton, PySCF, WIEN2k
             # (Qiskit depends on PySCF)
-            'libxc': (r'[23]\.', ['ABINIT-', 'AtomPAW-', 'CP2K-', 'GPAW-', 'PySCF-', 'Qiskit-', 'WIEN2k-']),
+            'libxc': (r'[23]\.', ['ABINIT-', 'AtomPAW-', 'CP2K-', 'GPAW-', 'horton-', 'PySCF-', 'Qiskit-', 'WIEN2k-']),
             # OPERA requires SAMtools 0.x
             'SAMtools': (r'0\.', ['ChimPipe-0.9.5', 'Cufflinks-2.2.1', 'OPERA-2.0.6']),
             # Kraken 1.x requires Jellyfish 1.x (Roary & metaWRAP depend on Kraken 1.x)
@@ -512,10 +514,11 @@ def check_sha256_checksums(self, changed_ecs):
         # for easyconfigs using the Bundle easyblock, this is a problem because the 'sources' easyconfig parameter
         # is updated in place (sources for components are added the 'parent' sources) in Bundle's __init__;
         # therefore, we need to reset 'sources' to an empty list here if Bundle is used...
-        # likewise for 'checksums'
+        # likewise for 'patches' and 'checksums'
         for ec in changed_ecs:
             if ec['easyblock'] == 'Bundle':
                 ec['sources'] = []
+                ec['patches'] = []
                 ec['checksums'] = []
 
         # filter out deprecated easyconfigs
@@ -527,11 +530,11 @@ def check_sha256_checksums(self, changed_ecs):
         checksum_issues = check_sha256_checksums(retained_changed_ecs, whitelist=whitelist)
         self.assertTrue(len(checksum_issues) == 0, "No checksum issues:\n%s" % '\n'.join(checksum_issues))
 
-    def check_python_packages(self, changed_ecs):
+    def check_python_packages(self, changed_ecs, added_ecs_filenames):
         """Several checks for easyconfigs that install (bundles of) Python packages."""
 
-        # MATLAB-Engine, PyTorch do not support installation with 'pip'
-        whitelist_pip = ['MATLAB-Engine-*', 'PyTorch-*']
+        # These packages do not support installation with 'pip'
+        whitelist_pip = [r'MATLAB-Engine-.*', r'PyTorch-.*', r'Meld-.*']
 
         failing_checks = []
 
@@ -540,6 +543,7 @@ def check_python_packages(self, changed_ecs):
             ec_fn = os.path.basename(ec.path)
             easyblock = ec.get('easyblock')
             exts_defaultclass = ec.get('exts_defaultclass')
+            exts_default_options = ec.get('exts_default_options', {})
 
             download_dep_fail = ec.get('download_dep_fail')
             exts_download_dep_fail = ec.get('exts_download_dep_fail')
@@ -569,7 +573,6 @@ def check_python_packages(self, changed_ecs):
                 else:
                     # both download_dep_fail and use_pip should be set via exts_default_options
                     # when installing Python packages as extensions
-                    exts_default_options = ec.get('exts_default_options', {})
                     for key in ['download_dep_fail', 'use_pip']:
                         if exts_default_options.get(key) is None:
                             failing_checks.append("'%s' set in exts_default_options in %s" % (key, ec_fn))
@@ -578,7 +581,19 @@ def check_python_packages(self, changed_ecs):
             # Tkinter is an exception, since its version always matches the Python version anyway
             if any(dep['name'] == 'Python' for dep in ec['dependencies']) and ec.name != 'Tkinter':
                 if not re.search(r'-Python-[23]\.[0-9]+\.[0-9]+', ec['versionsuffix']):
-                    failing_checks.append("'-Python-%%(pyver)s' included in versionsuffix in %s" % ec_fn)
+                    msg = "'-Python-%%(pyver)s' included in versionsuffix in %s" % ec_fn
+                    # This is only a failure for newly added ECs, not for existing ECS
+                    # As that would probably break many ECs
+                    if ec_fn in added_ecs_filenames:
+                        failing_checks.append(msg)
+                    else:
+                        print('\nNote: Failed non-critical check: ' + msg)
+
+            # require that running of "pip check" during sanity check is enabled via sanity_pip_check
+            if use_pip and easyblock in ['PythonBundle', 'PythonPackage']:
+                sanity_pip_check = ec.get('sanity_pip_check') or exts_default_options.get('sanity_pip_check')
+                if not sanity_pip_check and not any(re.match(regex, ec_fn) for regex in whitelist_pip):
+                    failing_checks.append("sanity_pip_check is enabled in %s" % ec_fn)
 
         self.assertFalse(failing_checks, '\n'.join(failing_checks))
 
@@ -588,8 +603,8 @@ def check_sanity_check_paths(self, changed_ecs):
         # PythonBundle & PythonPackage already have a decent customised sanity_check_paths
         # BuildEnv, ModuleRC and Toolchain easyblocks doesn't install anything so there is nothing to check.
         whitelist = ['CrayToolchain', 'ModuleRC', 'PythonBundle', 'PythonPackage', 'Toolchain', 'BuildEnv']
-        # GCC is just a bundle of GCCcore+binutils
-        bundles_whitelist = ['GCC']
+        # Autotools & (recent) GCC are just bundles (Autotools: Autoconf+Automake+libtool, GCC: GCCcore+binutils)
+        bundles_whitelist = ['Autotools', 'GCC']
 
         failing_checks = []
 
@@ -633,7 +648,7 @@ def check_https(self, changed_ecs):
             for http_url in http_regex.findall(ec_txt):
                 https_url = http_url.replace('http://', 'https://')
                 try:
-                    https_url_works = bool(urlopen(https_url))
+                    https_url_works = bool(urlopen(https_url, timeout=5))
                 except Exception:
                     https_url_works = False
 
@@ -644,6 +659,11 @@ def check_https(self, changed_ecs):
 
     def test_changed_files_pull_request(self):
         """Specific checks only done for the (easyconfig) files that were changed in a pull request."""
+        def get_eb_files_from_diff(diff_filter):
+            cmd = "git diff --name-only --diff-filter=%s %s...HEAD" % (diff_filter, target_branch)
+            out, ec = run_cmd(cmd, simple=False)
+            return [os.path.basename(f) for f in out.strip().split('\n') if f.endswith('.eb')]
+
 
         # $TRAVIS_PULL_REQUEST should be a PR number, otherwise we're not running tests for a PR
         travis_pr_test = re.match('^[0-9]+$', os.environ.get('TRAVIS_PULL_REQUEST', '(none)'))
@@ -673,16 +693,18 @@ def test_changed_files_pull_request(self):
                 cwd = change_dir(top_dir)
 
                 # get list of changed easyconfigs
-                cmd = "git diff --name-only --diff-filter=AM %s...HEAD" % target_branch
-                out, ec = run_cmd(cmd, simple=False)
-                changed_ecs_filenames = [os.path.basename(f) for f in out.strip().split('\n') if f.endswith('.eb')]
-                print("\nList of changed easyconfig files in this PR: %s" % '\n'.join(changed_ecs_filenames))
+                changed_ecs_filenames = get_eb_files_from_diff(diff_filter='M')
+                added_ecs_filenames = get_eb_files_from_diff(diff_filter='A')
+                if changed_ecs_filenames:
+                    print("\nList of changed easyconfig files in this PR: %s" % '\n'.join(changed_ecs_filenames))
+                if added_ecs_filenames:
+                    print("\nList of added easyconfig files in this PR: %s" % '\n'.join(added_ecs_filenames))
 
                 change_dir(cwd)
 
                 # grab parsed easyconfigs for changed easyconfig files
                 changed_ecs = []
-                for ec_fn in changed_ecs_filenames:
+                for ec_fn in changed_ecs_filenames + added_ecs_filenames:
                     match = None
                     for ec in EasyConfigTest.parsed_easyconfigs:
                         if os.path.basename(ec['spec']) == ec_fn:
@@ -706,7 +728,7 @@ def test_changed_files_pull_request(self):
 
                 # run checks on changed easyconfigs
                 self.check_sha256_checksums(changed_ecs)
-                self.check_python_packages(changed_ecs)
+                self.check_python_packages(changed_ecs, added_ecs_filenames)
                 self.check_sanity_check_paths(changed_ecs)
                 self.check_https(changed_ecs)
 
diff --git a/test/easyconfigs/styletests.py b/test/easyconfigs/styletests.py
--- a/test/easyconfigs/styletests.py
+++ b/test/easyconfigs/styletests.py
@@ -1,5 +1,5 @@
 ##
-# Copyright 2016-2019 Ghent University
+# Copyright 2016-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/easyconfigs/suite.py b/test/easyconfigs/suite.py
--- a/test/easyconfigs/suite.py
+++ b/test/easyconfigs/suite.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 ##
-# Copyright 2012-2019 Ghent University
+# Copyright 2012-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),