Repository: easybuilders/easybuild-easyconfigs

Issue:
CVS is not included in EasyBuild releases
The whole folder https://github.com/easybuilders/easybuild-easyconfigs/tree/develop/easybuild/easyconfigs/c/CVS is removed from EasyBuild source distributions.

This happens as part of `python setup.py sdist`.

https://docs.python.org/2/distutils/sourcedist.html#principle
> exclude the entire build tree, and any RCS, CVS, .svn, .hg, .git, .bzr and _darcs directories

https://packaging.python.org/guides/using-manifest-in/
>After processing the MANIFEST.in file, setuptools removes the build/ directory as well as any directories named RCS, CVS, or .svn from the sdist, and it adds a PKG-INFO file and an *.egg-info directory. This behavior cannot be changed with MANIFEST.in.



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = '4.1.1'
+VERSION = '4.2.0'
 
 MAJ_VER = VERSION.split('.')[0]
 MAJMIN_VER = '.'.join(VERSION.split('.')[0:2])


Test patch:
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -1,7 +1,7 @@
 name: easyconfigs unit tests
 on: [push, pull_request]
 jobs:
-  build:
+  test-suite:
     runs-on: ubuntu-18.04
     strategy:
       matrix:
@@ -20,7 +20,7 @@ jobs:
             module_syntax: Tcl
       fail-fast: false
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
 
     - name: set up Python
       uses: actions/setup-python@v1
@@ -30,10 +30,9 @@ jobs:
 
     - name: install OS & Python packages
       run: |
-        # apt-get update is disabled for now, because it was failing in GitHub since Fri Dec 13th 2019,
-        # due to 'Conflicting distribution' issue
+        # disable apt-get update, we don't really need it,
+        # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
         # sudo apt-get update
-        #
         # for modules tool
         sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
@@ -115,18 +114,64 @@ jobs:
           unset PYTHONPATH
 
           # install easyconfigs via distribution package
-          python setup.py sdist
+          python setup.py sdist > /dev/null
           ls dist
-          pip install dist/easybuild-easyconfigs*tar.gz
+          pip install dist/easybuild-easyconfigs*tar.gz > /dev/null
 
           # robot-paths value should not be empty, but have an entry that includes easybuild/easyconfigs subdir
+          echo "eb --show-config"
           eb --show-config | tee eb_show_config.out
           grep "^robot-paths .*/easybuild/easyconfigs" eb_show_config.out
+
           # check whether some specific easyconfig files are found
+          echo "eb --search 'TensorFlow-1.14.*.eb'"
           eb --search 'TensorFlow-1.14.*.eb' | tee eb_search_TF.out
           grep '/TensorFlow-1.14.0-foss-2019a-Python-3.7.2.eb$' eb_search_TF.out
-          eb --search '^foss-2018b.eb' | tee eb_search_foss.out
-          grep '/foss-2018b.eb$' eb_search_foss.out
 
-          # try installing bzip2 with system toolchain (requires EB_bzip2 easyblock + easyconfig)
-          eb --prefix /tmp/$USER/$GITHUB_SHA bzip2-1.0.6.eb
+          echo "eb --search '^foss-2019b.eb'"
+          eb --search '^foss-2019b.eb' | tee eb_search_foss.out
+          grep '/foss-2019b.eb$' eb_search_foss.out
+
+          # make sure CVS easyconfigs are included in installation (cfr. issue #10325)
+          echo "Searching for CVS easyconfigs..."
+          eb --search '^CVS-' | grep '/CVS-'
+
+          # try installing M4 with system toolchain (requires ConfigureMake easyblock + easyconfig)
+          eb --prefix /tmp/$USER/$GITHUB_SHA M4-1.4.18.eb
+
+  test-sdist:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python: [2.7, 3.6, 3.7]
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: set up Python
+      uses: actions/setup-python@v1
+
+    - name: Create source distribution
+      run: python setup.py sdist
+
+    - name: Inspect files included in source distribution
+      working-directory: dist
+      run: |
+          tar xfz easybuild-easyconfigs*tar.gz
+          cd easybuild-easyconfigs-*/
+
+          # .git folder should not be there in source tarball
+          dot_git_files=$(find . -name .git)
+          if [ -n "$dot_git_files" ]; then
+            echo "Found .git folders in source tarball: $dot_git_files" && false
+          else
+            echo "No .git folders found in source tarball: OK"
+          fi
+
+          # CVS easyconfigs must be included in source tarball,
+          # see https://github.com/easybuilders/easybuild-easyconfigs/issues/10325
+          cvs_easyconfigs=$(find . -name 'CVS-*.eb')
+          if [ -z "$cvs_easyconfigs" ]; then
+            echo "CVS easyconfigs not found" && false
+          else
+            echo "Found CVS easyconfigs: $cvs_easyconfigs"
+          fi
diff --git a/easybuild/easyconfigs/b/BerkeleyGW/BerkeleyGW-2.1.0_tests.patch b/easybuild/easyconfigs/b/BerkeleyGW/BerkeleyGW-2.1.0_tests.patch
new file mode 100644
--- /dev/null
+++ b/easybuild/easyconfigs/b/BerkeleyGW/BerkeleyGW-2.1.0_tests.patch
@@ -0,0 +1,42 @@
+* enable parallel tests
+* allow test scripts to inherit environment
+* slightly raise tolerance in one test
+author: Miguel Dias Costa (National University of Singapore)
+--- Makefile.orig	2019-07-23 17:40:16.489733901 +0800
++++ Makefile	2019-07-23 17:40:33.656497133 +0800
+@@ -157,7 +157,7 @@
+ endif
+ 
+ check: all
+-	cd testsuite && $(MAKE) check
++	cd testsuite && $(MAKE) check && $(MAKE) check-parallel
+ 
+ check-save: all
+ 	cd testsuite && $(MAKE) check-save
+--- testsuite/run_testsuite.sh.orig	2019-07-25 14:48:12.000000000 +0800
++++ testsuite/run_testsuite.sh	2019-07-25 14:47:50.000000000 +0800
+@@ -1,4 +1,4 @@
+-#!/bin/bash -l
++#!/bin/bash
+ #
+ # Copyright (C) 2005-2009 Heiko Appel, David Strubbe
+ #
+--- MeanField/Utilities/mf_convert_wrapper.sh.orig	2019-07-26 09:52:27.648341000 +0800
++++ MeanField/Utilities/mf_convert_wrapper.sh	2019-07-26 09:52:39.922114228 +0800
+@@ -1,4 +1,4 @@
+-#!/bin/bash -l
++#!/bin/bash
+ 
+ # David Strubbe, October 2010
+ # Wrapper for mf_convert.x
+--- testsuite/GaAs-EPM/GaAs.test.orig	2019-07-26 10:45:19.798520000 +0800
++++ testsuite/GaAs-EPM/GaAs.test	2019-07-26 10:45:37.753775275 +0800
+@@ -32,7 +32,7 @@
+ Output : WFN.out
+ Input  : WFN.in PIPE
+ 
+-Precision : 8e-15
++Precision : 9e-15
+ match ; Eigenvalue  1 at k-pt 1   ; GREP(WFN.out,   "kpoint          1", 2,  1); -0.2710614199849328
+ match ; Eigenvalue 10 at k-pt 1   ; GREP(WFN.out,   "kpoint          1", 2, 10);  1.2565373697755460
+ match ; Eigenvalue 18 at k-pt 2   ; GREP(WFN.out,   "kpoint          2", 2, 18);  2.1322637363008994
diff --git a/easybuild/easyconfigs/r/RDKit/RDKit-2019.09.3_fix_tests.patch b/easybuild/easyconfigs/r/RDKit/RDKit-2019.09.3_fix_tests.patch
new file mode 100644
--- /dev/null
+++ b/easybuild/easyconfigs/r/RDKit/RDKit-2019.09.3_fix_tests.patch
@@ -0,0 +1,76 @@
+Fix failing test for correct but permutated atom sets
+Author: Samuel Moors, Vrije Universiteit Brussel (VUB)
+diff -ur rdkit-Release_2019_09_3.orig/Code/GraphMol/Wrap/rough_test.py rdkit-Release_2019_09_3/Code/GraphMol/Wrap/rough_test.py
+--- rdkit-Release_2019_09_3.orig/Code/GraphMol/Wrap/rough_test.py	2020-01-09 18:20:46.000000000 +0100
++++ rdkit-Release_2019_09_3/Code/GraphMol/Wrap/rough_test.py	2020-03-14 16:06:22.537832599 +0100
+@@ -3889,7 +3889,8 @@
+     self.assertEqual(matches, ((1, 2, 3), (1, 3, 2)))
+     matches = resMolSuppl.GetSubstructMatches(query, uniquify=True)
+     self.assertEqual(len(matches), 1)
+-    self.assertEqual(matches, ((1, 2, 3), ))
++    # self.assertEqual(matches, ((1, 2, 3), ))
++    self.assertEqual(tuple(set(x) for x in matches), ({1, 2, 3}, ))
+     matches = resMolSuppl.GetSubstructMatches(query, uniquify=False)
+     self.assertEqual(len(matches), 2)
+     self.assertEqual(matches, ((1, 2, 3), (1, 3, 2)))
+@@ -3899,13 +3900,15 @@
+     self.assertEqual(matches, ((1, 2, 3), (1, 3, 2)))
+     matches = mol.GetSubstructMatches(query, uniquify=True)
+     self.assertEqual(len(matches), 1)
+-    self.assertEqual(matches, ((1, 2, 3), ))
++    # self.assertEqual(matches, ((1, 2, 3), ))
++    self.assertEqual(tuple(set(x) for x in matches), ({1, 2, 3}, ))
+     matches = resMolSuppl.GetSubstructMatches(query, uniquify=False)
+     self.assertEqual(len(matches), 2)
+     self.assertEqual(matches, ((1, 2, 3), (1, 3, 2)))
+     matches = resMolSuppl.GetSubstructMatches(query, uniquify=True)
+     self.assertEqual(len(matches), 1)
+-    self.assertEqual(matches, ((1, 2, 3), ))
++    # self.assertEqual(matches, ((1, 2, 3), ))
++    self.assertEqual(tuple(set(x) for x in matches), ({1, 2, 3}, ))
+ 
+   def testSubstructMatchDMAP(self):
+     mol = Chem.MolFromSmiles('C(C)Nc1cc[nH+]cc1')
+@@ -3964,7 +3967,8 @@
+                                (320, 321, 326), (320, 326, 321)))
+     matches = resMolSupplST.GetSubstructMatches(carboxylateQuery, uniquify=True)
+     self.assertEqual(len(matches), 3)
+-    self.assertEqual(matches, ((166, 167, 168), (298, 299, 300), (320, 321, 326)))
++    # self.assertEqual(matches, ((166, 167, 168), (298, 299, 300), (320, 321, 326)))
++    self.assertEqual(tuple(set(x) for x in matches), ({166, 167, 168}, {298, 299, 300}, {320, 321, 326}))
+     matches = resMolSupplST.GetSubstructMatches(guanidiniumQuery)
+     self.assertEqual(len(matches), 8)
+     self.assertEqual(matches, ((66, 67, 68, 69), (66, 67, 69, 68), (68, 67, 69, 66),
+@@ -3972,7 +3976,8 @@
+                                (125, 124, 126, 123), (126, 124, 125, 123)))
+     matches = resMolSupplST.GetSubstructMatches(guanidiniumQuery, uniquify=True)
+     self.assertEqual(len(matches), 2)
+-    self.assertEqual(matches, ((66, 67, 69, 68), (123, 124, 126, 125)))
++    # self.assertEqual(matches, ((66, 67, 69, 68), (123, 124, 126, 125)))
++    self.assertEqual(tuple(set(x) for x in matches), ({66, 67, 69, 68}, {123, 124, 126, 125}))
+     btList2ST = getBtList2(resMolSupplST)
+     self.assertTrue(btList2ST)
+     resMolSupplMT = Chem.ResonanceMolSupplier(crambin)
+@@ -3991,7 +3996,8 @@
+                                  (320, 321, 326), (320, 326, 321)))
+       matches = suppl.GetSubstructMatches(carboxylateQuery, uniquify=True, numThreads=0)
+       self.assertEqual(len(matches), 3)
+-      self.assertEqual(matches, ((166, 167, 168), (298, 299, 300), (320, 321, 326)))
++      # self.assertEqual(matches, ((166, 167, 168), (298, 299, 300), (320, 321, 326)))
++      self.assertEqual(tuple(set(x) for x in matches), ({166, 167, 168}, {298, 299, 300}, {320, 321, 326}))
+       matches = suppl.GetSubstructMatches(guanidiniumQuery, numThreads=0)
+       self.assertEqual(len(matches), 8)
+       self.assertEqual(matches, ((66, 67, 68, 69), (66, 67, 69, 68), (68, 67, 69, 66),
+@@ -3999,7 +4005,8 @@
+                                  (125, 124, 126, 123), (126, 124, 125, 123)))
+       matches = suppl.GetSubstructMatches(guanidiniumQuery, uniquify=True, numThreads=0)
+       self.assertEqual(len(matches), 2)
+-      self.assertEqual(matches, ((66, 67, 69, 68), (123, 124, 126, 125)))
++      # self.assertEqual(matches, ((66, 67, 69, 68), (123, 124, 126, 125)))
++      self.assertEqual(tuple(set(x) for x in matches), ({66, 67, 69, 68}, {123, 124, 126, 125}))
+ 
+   def testGitHUb1166(self):
+     mol = Chem.MolFromSmiles('NC(=[NH2+])c1ccc(cc1)C(=O)[O-]')
+Only in rdkit-Release_2019_09_3.orig/Code/GraphMol/Wrap/test_data: outNCI_few.sdf
+Only in rdkit-Release_2019_09_3.orig/Code/GraphMol/Wrap/test_data: outSmiles.txt
+Only in rdkit-Release_2019_09_3/rdkit: __pycache__
diff --git a/test/easyconfigs/easyconfigs.py b/test/easyconfigs/easyconfigs.py
--- a/test/easyconfigs/easyconfigs.py
+++ b/test/easyconfigs/easyconfigs.py
@@ -57,6 +57,7 @@
 from easybuild.tools.robot import check_conflicts, resolve_dependencies
 from easybuild.tools.run import run_cmd
 from easybuild.tools.options import set_tmpdir
+from easybuild.tools.utilities import nub
 
 
 # indicates whether all the single tests are OK,
@@ -210,7 +211,7 @@ def check_dep_vars(self, dep, dep_vars):
 
         # for some dependencies, we allow exceptions for software that depends on a particular version,
         # as long as that's indicated by the versionsuffix
-        if dep in ['ASE', 'Boost', 'Java', 'Lua', 'PLUMED', 'R', 'TensorFlow'] and len(dep_vars) > 1:
+        if dep in ['ASE', 'Boost', 'Java', 'Lua', 'PLUMED', 'PyTorch', 'R', 'TensorFlow'] and len(dep_vars) > 1:
             for key in list(dep_vars):
                 dep_ver = re.search('^version: (?P<ver>[^;]+);', key).group('ver')
                 # use version of Java wrapper rather than full Java version
@@ -243,14 +244,21 @@ def check_dep_vars(self, dep, dep_vars):
         old_dep_versions = {
             # libxc 2.x or 3.x is required by ABINIT, AtomPAW, CP2K, GPAW, horton, PySCF, WIEN2k
             # (Qiskit depends on PySCF)
-            'libxc': (r'[23]\.', ['ABINIT-', 'AtomPAW-', 'CP2K-', 'GPAW-', 'horton-', 'PySCF-', 'Qiskit-', 'WIEN2k-']),
+            'libxc': (r'[23]\.', [r'ABINIT-', r'AtomPAW-', r'CP2K-', r'GPAW-', r'horton-', r'PySCF-',
+                                  r'Qiskit-', r'WIEN2k-']),
             # OPERA requires SAMtools 0.x
-            'SAMtools': (r'0\.', ['ChimPipe-0.9.5', 'Cufflinks-2.2.1', 'OPERA-2.0.6']),
+            'SAMtools': (r'0\.', [r'ChimPipe-0\.9\.5', r'Cufflinks-2\.2\.1', r'OPERA-2\.0\.6', r'CGmapTools-0\.1\.2',
+                                  r'BatMeth2-2\.1']),
             # Kraken 1.x requires Jellyfish 1.x (Roary & metaWRAP depend on Kraken 1.x)
-            'Jellyfish': (r'1\.', ['Kraken-1.', 'Roary-3.12.0', 'metaWRAP-1.2']),
+            'Jellyfish': (r'1\.', [r'Kraken-1\.', r'Roary-3\.12\.0', r'metaWRAP-1\.2']),
             # EMAN2 2.3 requires Boost(.Python) 1.64.0
-            'Boost': ('1.64.0;', ['Boost.Python-1.64.0-', 'EMAN2-2.3-']),
-            'Boost.Python': ('1.64.0;', ['EMAN2-2.3-']),
+            'Boost': ('1.64.0;', [r'Boost.Python-1\.64\.0-', r'EMAN2-2\.3-']),
+            'Boost.Python': ('1.64.0;', [r'EMAN2-2\.3-']),
+            # numba 0.47.x requires LLVM 7.x or 8.x (see https://github.com/numba/llvmlite#compatibility)
+            # both scVelo and Python-Geometric depend on numba
+            'LLVM': (r'8\.', [r'numba-0\.47\.0-', r'scVelo-0\.1\.24-', r'PyTorch-Geometric-1\.[34]\.2']),
+            # medaka 0.11.4/0.12.0requires recent TensorFlow <= 1.14 (and Python 3.6)
+            'TensorFlow': ('1.13.1;', ['medaka-0.11.4-', 'medaka-0.12.0-']),
         }
         if dep in old_dep_versions and len(dep_vars) > 1:
             for key in list(dep_vars):
@@ -258,9 +266,15 @@ def check_dep_vars(self, dep, dep_vars):
                 # filter out known old dependency versions
                 if re.search('^version: %s' % version_pattern, key):
                     # only filter if the easyconfig using this dep variants is known
-                    if all(any(x.startswith(p) for p in parents) for x in dep_vars[key]):
+                    if all(any(re.search(p, x) for p in parents) for x in dep_vars[key]):
                         dep_vars.pop(key)
 
+        # filter out ELSI variants with -PEXSI suffix
+        if dep == 'ELSI' and len(dep_vars) > 1:
+            pexsi_vsuff_vars = [v for v in dep_vars.keys() if v.endswith('versionsuffix: -PEXSI')]
+            if len(pexsi_vsuff_vars) == 1:
+                dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != pexsi_vsuff_vars[0])
+
         # only single variant is always OK
         if len(dep_vars) == 1:
             res = True
@@ -508,7 +522,7 @@ def check_sha256_checksums(self, changed_ecs):
 
         # list of software for which checksums can not be required,
         # e.g. because 'source' files need to be constructed manually
-        whitelist = ['Kent_tools-*', 'MATLAB-*', 'OCaml-*']
+        whitelist = ['Kent_tools-*', 'MATLAB-*', 'OCaml-*', 'OpenFOAM-Extend-4.1-*']
 
         # the check_sha256_checksums function (again) creates an EasyBlock instance
         # for easyconfigs using the Bundle easyblock, this is a problem because the 'sources' easyconfig parameter
@@ -516,7 +530,7 @@ def check_sha256_checksums(self, changed_ecs):
         # therefore, we need to reset 'sources' to an empty list here if Bundle is used...
         # likewise for 'patches' and 'checksums'
         for ec in changed_ecs:
-            if ec['easyblock'] == 'Bundle':
+            if ec['easyblock'] in ['Bundle', 'PythonBundle']:
                 ec['sources'] = []
                 ec['patches'] = []
                 ec['checksums'] = []
@@ -630,6 +644,12 @@ def check_https(self, changed_ecs):
             'p4vasp',  # https://www.p4vasp.at doesn't work
             'ITSTool',  # https://itstool.org/ doesn't work
             'UCX-',  # bad certificate for https://www.openucx.org
+            'MUMPS',  # https://mumps.enseeiht.fr doesn't work
+        ]
+        url_whitelist = [
+            # https:// doesn't work, results in index page being downloaded instead
+            # (see https://github.com/easybuilders/easybuild-easyconfigs/issues/9692)
+            'http://isl.gforge.inria.fr',
         ]
 
         http_regex = re.compile('http://[^"\'\n]+', re.M)
@@ -646,6 +666,11 @@ def check_https(self, changed_ecs):
             ec_txt = '\n'.join(l for l in ec.rawtxt.split('\n') if not l.startswith('#'))
 
             for http_url in http_regex.findall(ec_txt):
+
+                # skip whitelisted http:// URLs
+                if any(http_url.startswith(x) for x in url_whitelist):
+                    continue
+
                 https_url = http_url.replace('http://', 'https://')
                 try:
                     https_url_works = bool(urlopen(https_url, timeout=5))
@@ -805,6 +830,20 @@ def template_easyconfig_test(self, spec):
     error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s"
     self.assertFalse(res, error_msg % res)
 
+    # check for redefined easyconfig parameters, there should be none...
+    param_def_regex = re.compile('^(?P<key>\w+)\s*=', re.M)
+    keys = param_def_regex.findall(ec.rawtxt)
+    redefined_keys = []
+    for key in sorted(nub(keys)):
+        cnt = keys.count(key)
+        if cnt > 1:
+            redefined_keys.append((key, cnt))
+
+    redefined_keys_error_msg = "There should be no redefined easyconfig parameters, found %d: " % len(redefined_keys)
+    redefined_keys_error_msg += ', '.join('%s (%d)' % x for x in redefined_keys)
+
+    self.assertFalse(redefined_keys, redefined_keys_error_msg)
+
     # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used
     old_urls = [
         'github.com/hpcugent/easybuild',