Repository: easybuilders/easybuild-framework

Issue:
Bootstrap script incompatible with python3
As observed in #3017 `bootstrap_eb.py` is not compatible with Python3. It relies on `/usr/bin/env python` being Python 2 (which is still the case on most but not all Linux distributions).

Fixing that is not a simple case of translating the file, as it included a base64'ed gzipped setuptools helper that is outdated.

One way that seems to work for stage1 is something akin too:
```
wget https://bootstrap.pypa.io/get-pip.py
export EB_TMPLOC=<sometmplocation like /tmp/tmpRG7iSQ/eb_stage1>
export PYTHONPATH=$EB_TMPLOC
python3 get-pip.py --prefix=$EB_TMPLOC
$EB_TMPLOC/bin/pip --prefix=$EB_TMPLOC easybuild
```
after that it can do stage2 as usual, since `eb` is installed now.

I'm just floating this as an idea since it's a complete redesign of the stage1 of the bootstrap.


Discussion:
I started working on this, and it's indeed a bit more work that just porting the script itself.

The good thing is that since EasyBuild 4.0 no longer requires `setuptools` at all, we can actually move get rid of the base64'd `distribute` installation script (and just check if `setuptools` is available in case somebody installs an older version of EasyBuild, rather than trying to install it ourselves which is a bit of a desperate attempt anyway).

That also means no longer using `easy_install` during stage 1 (which is provided by `setuptools`), we should just use `setup.py install` (which implies downloading and unpacking the source tarballs first).

Kickstarted the effort in #3028...

Resolving patch:
diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py
--- a/easybuild/scripts/bootstrap_eb.py
+++ b/easybuild/scripts/bootstrap_eb.py
@@ -40,6 +40,7 @@
 (via http://dubroy.com/blog/so-you-want-to-install-a-python-package/)
 """
 
+import codecs
 import copy
 import glob
 import os
@@ -49,12 +50,19 @@
 import sys
 import tempfile
 import traceback
-import urllib2
 from distutils.version import LooseVersion
 from hashlib import md5
+from platform import python_version
 
+IS_PY3 = sys.version_info[0] == 3
 
-EB_BOOTSTRAP_VERSION = '20190922.01'
+if not IS_PY3:
+    import urllib2 as std_urllib
+else:
+    import urllib.request as std_urllib
+
+
+EB_BOOTSTRAP_VERSION = '20200203.01'
 
 # argparse preferrred, optparse deprecated >=2.7
 HAVE_ARGPARSE = False
@@ -68,7 +76,9 @@
 
 VSC_BASE = 'vsc-base'
 VSC_INSTALL = 'vsc-install'
-EASYBUILD_PACKAGES = [VSC_INSTALL, VSC_BASE, 'easybuild-framework', 'easybuild-easyblocks', 'easybuild-easyconfigs']
+# Python 3 is not supported by the vsc-* packages
+EASYBUILD_PACKAGES = (([] if IS_PY3 else [VSC_INSTALL, VSC_BASE]) +
+                      ['easybuild-framework', 'easybuild-easyblocks', 'easybuild-easyconfigs'])
 
 STAGE1_SUBDIR = 'eb_stage1'
 
@@ -127,8 +137,10 @@ def error(msg, exit=True):
 
 def mock_stdout_stderr():
     """Mock stdout/stderr channels"""
-    # cStringIO is only available in Python 2
-    from cStringIO import StringIO
+    try:
+        from cStringIO import StringIO
+    except ImportError:
+        from io import StringIO
     orig_stdout, orig_stderr = sys.stdout, sys.stderr
     sys.stdout.flush()
     sys.stdout = StringIO()
@@ -324,7 +336,7 @@ def check_setuptools():
 
     # check setuptools version
     try:
-        os.system(cmd_tmpl % "import setuptools; print setuptools.__version__")
+        os.system(cmd_tmpl % "import setuptools; print(setuptools.__version__)")
         setuptools_ver = LooseVersion(open(outfile).read().strip())
         debug("Found setuptools version %s" % setuptools_ver)
 
@@ -336,7 +348,7 @@ def check_setuptools():
         debug("Failed to check setuptools version: %s" % err)
         res = False
 
-    os.system(cmd_tmpl % "from setuptools.command import easy_install; print easy_install.__file__")
+    os.system(cmd_tmpl % "from setuptools.command import easy_install; print(easy_install.__file__)")
     out = open(outfile).read().strip()
     debug("Location of setuptools' easy_install module: %s" % out)
     if 'setuptools/command/easy_install' not in out:
@@ -344,7 +356,7 @@ def check_setuptools():
         res = False
 
     if res is None:
-        os.system(cmd_tmpl % "import setuptools; print setuptools.__file__")
+        os.system(cmd_tmpl % "import setuptools; print(setuptools.__file__)")
         setuptools_loc = open(outfile).read().strip()
         res = os.path.dirname(os.path.dirname(setuptools_loc))
         debug("Location of setuptools installation: %s" % res)
@@ -523,27 +535,32 @@ def stage1(tmpdir, sourcepath, distribute_egg_dir, forcedversion):
         # install meta-package easybuild from PyPI
         if forcedversion:
             cmd.append('easybuild==%s' % forcedversion)
+        elif IS_PY3:
+            cmd.append('easybuild>=4.0')  # Python 3 support added in EasyBuild 4
         else:
             cmd.append('easybuild')
 
-        # install vsc-base again at the end, to avoid that the one available on the system is used instead
-        post_vsc_base = cmd[:]
-        post_vsc_base[-1] = VSC_BASE + '<2.9.0'
+        if not IS_PY3:
+            # install vsc-base again at the end, to avoid that the one available on the system is used instead
+            post_vsc_base = cmd[:]
+            post_vsc_base[-1] = VSC_BASE + '<2.9.0'
 
     if not print_debug:
         cmd.insert(0, '--quiet')
 
-    # install vsc-install version prior to 0.11.4, where mock was introduced as a dependency
-    # workaround for problem reported in https://github.com/easybuilders/easybuild-framework/issues/2712
-    # also stick to vsc-base < 2.9.0 to avoid requiring 'future' Python package as dependency
-    for pkg in [VSC_INSTALL + '<0.11.4', VSC_BASE + '<2.9.0']:
-        precmd = cmd[:-1] + [pkg]
-        info("running pre-install command 'easy_install %s'" % (' '.join(precmd)))
-        run_easy_install(precmd)
+    # There is no support for Python3 in the older vsc-* packages and EasyBuild 4 includes working versions of vsc-*
+    if not IS_PY3:
+        # install vsc-install version prior to 0.11.4, where mock was introduced as a dependency
+        # workaround for problem reported in https://github.com/easybuilders/easybuild-framework/issues/2712
+        # also stick to vsc-base < 2.9.0 to avoid requiring 'future' Python package as dependency
+        for pkg in [VSC_INSTALL + '<0.11.4', VSC_BASE + '<2.9.0']:
+            precmd = cmd[:-1] + [pkg]
+            info("running pre-install command 'easy_install %s'" % (' '.join(precmd)))
+            run_easy_install(precmd)
 
     info("installing EasyBuild with 'easy_install %s'\n" % (' '.join(cmd)))
     syntax_error_note = '\n'.join([
-        "Note: a 'SyntaxError' may be reported for the easybuild/tools/py2vs3/py3.py module.",
+        "Note: a 'SyntaxError' may be reported for the easybuild/tools/py2vs3/py%s.py module." % ('3', '2')[IS_PY3],
         "You can safely ignore this message, it will not affect the functionality of the EasyBuild installation.",
         '',
     ])
@@ -632,8 +649,13 @@ def stage1(tmpdir, sourcepath, distribute_egg_dir, forcedversion):
     # make sure we're getting the expected EasyBuild packages
     import easybuild.framework
     import easybuild.easyblocks
-    import vsc.utils.fancylogger
-    for pkg in [easybuild.framework, easybuild.easyblocks, vsc.utils.fancylogger]:
+    pkgs_to_check = [easybuild.framework, easybuild.easyblocks]
+    # vsc is part of EasyBuild 4
+    if LooseVersion(eb_version) < LooseVersion('4'):
+        import vsc.utils.fancylogger
+        pkgs_to_check.append(vsc.utils.fancylogger)
+
+    for pkg in pkgs_to_check:
         if tmpdir not in pkg.__file__:
             error("Found another %s than expected: %s" % (pkg.__name__, pkg.__file__))
         else:
@@ -698,8 +720,8 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
         # determine download URL via PyPI's 'simple' API
         pkg_simple = None
         try:
-            pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read()
-        except (urllib2.URLError, urllib2.HTTPError) as err:
+            pkg_simple = std_urllib.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read()
+        except (std_urllib.URLError, std_urllib.HTTPError) as err:
             # failing to figure out the package download URl may be OK when source tarballs are provided
             if sourcepath:
                 info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg)
@@ -707,6 +729,8 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
                 raise err
 
         if pkg_simple:
+            if IS_PY3:
+                pkg_simple = pkg_simple.decode('utf-8')
             pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename)
             res = pkg_url_part_regex.search(pkg_simple)
             if res:
@@ -827,6 +851,8 @@ def main():
     """Main script: bootstrap EasyBuild in stages."""
 
     self_txt = open(__file__).read()
+    if IS_PY3:
+        self_txt = self_txt.encode('utf-8')
     info("EasyBuild bootstrap script (version %s, MD5: %s)" % (EB_BOOTSTRAP_VERSION, md5(self_txt).hexdigest()))
     info("Found Python %s\n" % '; '.join(sys.version.split('\n')))
 
@@ -866,6 +892,9 @@ def main():
     forcedversion = EASYBUILD_BOOTSTRAP_FORCE_VERSION
     if forcedversion:
         info("Forcing specified version %s..." % forcedversion)
+        if IS_PY3 and LooseVersion(forcedversion) < LooseVersion('4'):
+            error('Python 3 support is only available with EasyBuild 4.x but you are trying to install EasyBuild %s'
+                  % forcedversion)
 
     # create temporary dir for temporary installations
     tmpdir = tempfile.mkdtemp()
@@ -982,10 +1011,12 @@ def main():
 """
 
 # check Python version
-if sys.version_info[0] != 2 or sys.version_info[1] < 6:
-    pyver = sys.version.split(' ')[0]
-    sys.stderr.write("ERROR: Incompatible Python version: %s (should be Python 2 >= 2.6)\n" % pyver)
-    sys.stderr.write("Please try again using 'python2 %s <prefix>'\n" % os.path.basename(__file__))
+loose_pyver = LooseVersion(python_version())
+min_pyver2 = LooseVersion('2.6')
+min_pyver3 = LooseVersion('3.5')
+if loose_pyver < min_pyver2 or (loose_pyver >= LooseVersion('3') and loose_pyver < min_pyver3):
+    sys.stderr.write("ERROR: Incompatible Python version: %s (should be Python 2 >= %s or Python 3 >= %s)\n"
+                     % (python_version(), min_pyver2, min_pyver3))
     sys.exit(1)
 
 # distribute_setup.py script (https://pypi.python.org/pypi/distribute)
@@ -1117,8 +1148,10 @@ def main():
 T4E5Gl7wpTxDXdQtzS1Hv52qHSilmOtEVO3IVjCdl5cgC5VC9T6CY1N4U4B0E1tltaqRtuYc/PyB
 i9tGe6+O/V0LCkGXvNkrKK2++u9qLFyTkO2sp7xSt/Bfil9os3SeOlY5fvv9mLcFj5zSNUqsRZfU
 7lwukTHLpfpLDH2GT+yCCf8D2cp1xw==
-
-""".decode("base64").decode("zlib")
+"""
+if IS_PY3:
+    DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.encode('ascii')
+DISTRIBUTE_SETUP_PY = codecs.decode(codecs.decode(DISTRIBUTE_SETUP_PY, "base64"), "zlib")
 
 # run main function as body of script
 main()


Test patch:
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -153,19 +153,15 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20190922.01 7927513e7448d886decfb1bb5daf840e85dc7367f57cc75e51b68f21fe109d53"
+          EB_BOOTSTRAP_EXPECTED="20200203.01 616bf3ce812c0844bf9ea3e690f9d88b394ed48f834ddb8424a73cf45fc64ea5"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
-          # test bootstrap script (only compatible with Python 2 for now)
-          if [[ ${{matrix.python}} =~ '2.' ]]; then
-              export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
-              python easybuild/scripts/bootstrap_eb.py $PREFIX
-              # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
-              unset PYTHONPATH
-              # simple sanity check on bootstrapped EasyBuild module (skip when testing with Python 3, for now)
-              module use $PREFIX/modules/all
-              module load EasyBuild
-              eb --version
-          else
-              echo "Testing of bootstrap script skipped when testing with Python ${{matrix.python}}"
-          fi
+          # test bootstrap script
+          export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
+          python easybuild/scripts/bootstrap_eb.py $PREFIX
+          # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
+          unset PYTHONPATH
+          # simple sanity check on bootstrapped EasyBuild module
+          module use $PREFIX/modules/all
+          module load EasyBuild
+          eb --version