Repository: certbot/certbot

Issue:
scl_source may not exist
As described in the comments following https://community.letsencrypt.org/t/certbot-failing-to-find-python-version-on-centos-6-10/105634/5, it's possible that `scl_source` does not exist on RHEL 6 based systems. To reproduce this in the CentOS 6 Docker image, run:
```
yum install epel-release -y
yum install scl-utils-20120229 -y
```
You can enable the SCL repository and install `rh-python36` after this and a newer version of `scl-utils` containing `scl_source` will not be installed.

I think we should fix this before rereleasing the code migrating people to Python 3.6 on SCL.

To do this, I think we can either try to make sure a new enough version of `scl-utils` is installed (although I'm not sure the best way to do that) or stop using `scl_source` and either manually modify our `PATH` or manually source `/opt/rh/rh-python36/enable`.


Discussion:

Resolving patch:
diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py
--- a/certbot/certbot/_internal/main.py
+++ b/certbot/certbot/_internal/main.py
@@ -1337,6 +1337,10 @@ def main(cli_args=None):
         if config.func != plugins_cmd:  # pylint: disable=comparison-with-callable
             raise
 
+    if sys.version_info[:2] == (3, 4):
+        logger.warning("Python 3.4 support will be dropped in the next release "
+                       "of Certbot - please upgrade your Python version to 3.5+.")
+
     set_displayer(config)
 
     # Reporter


Test patch:
diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh
--- a/letsencrypt-auto-source/tests/centos6_tests.sh
+++ b/letsencrypt-auto-source/tests/centos6_tests.sh
@@ -1,20 +1,22 @@
 #!/bin/bash
 set -e
 # Start by making sure your system is up-to-date:
-yum update -y > /dev/null
-yum install -y centos-release-scl > /dev/null
-yum install -y python27 > /dev/null 2> /dev/null
+yum update -y >/dev/null
+yum install -y centos-release-scl >/dev/null
+yum install -y python27 >/dev/null 2>/dev/null
 
+LE_AUTO_PY_34="certbot/letsencrypt-auto-source/letsencrypt-auto_py_34"
 LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto"
 
-echo ""
+# Last version of certbot-auto that was bootstraping Python 3.4 for CentOS 6 users
+INITIAL_CERTBOT_VERSION_PY34="certbot 0.38.0"
 
 # we're going to modify env variables, so do this in a subshell
 (
 # ensure CentOS6 32bits is not supported anymore, and so certbot is not installed
 export UNAME_FAKE_32BITS=true
 if ! "$LE_AUTO" 2>&1 | grep -q "Certbot cannot be installed."; then
-  echo "On CentOS 32 bits, certbot-auto installed certbot."
+  echo "ERROR: certbot-auto installed certbot on 32-bit CentOS."
   exit 1
 fi
 )
@@ -23,97 +25,149 @@ echo "PASSED: On CentOS 6 32 bits, certbot-auto refused to install certbot."
 
 # we're going to modify env variables, so do this in a subshell
 (
-source /opt/rh/python27/enable
+  . /opt/rh/python27/enable
+
+  # ensure python 3 isn't installed
+  if python3 --version 2> /dev/null; then
+    echo "ERROR: Python3 is already installed."
+    exit 1
+  fi
+
+  # ensure python2.7 is available
+  if ! python2.7 --version 2> /dev/null; then
+    echo "ERROR: Python2.7 is not available."
+    exit 1
+  fi
+
+  # bootstrap, but don't install python 3.
+  "$LE_AUTO" --no-self-upgrade -n --version > /dev/null 2> /dev/null
+
+  # ensure python 3 isn't installed
+  if python3 --version 2> /dev/null; then
+    echo "ERROR: letsencrypt-auto installed Python3 even though Python2.7 is present."
+    exit 1
+  fi
+
+  echo "PASSED: Did not upgrade to Python3 when Python2.7 is present."
+)
 
-# ensure python 3 isn't installed
-if python3 --version 2> /dev/null; then
-  echo "Python3 is already installed."
+# ensure python2.7 isn't available
+if python2.7 --version 2> /dev/null; then
+  echo "ERROR: Python2.7 is still available."
   exit 1
 fi
 
-# ensure python2.7 is available
-if ! python2.7 --version 2> /dev/null; then
-  echo "Python2.7 is not available."
+# Skip self upgrade due to Python 3 not being available.
+if ! "$LE_AUTO" 2>&1 | grep -q "WARNING: couldn't find Python"; then
+  echo "ERROR: Python upgrade failure warning not printed!"
   exit 1
 fi
 
-# bootstrap, but don't install python 3.
-"$LE_AUTO" --no-self-upgrade -n --version > /dev/null 2> /dev/null
+# bootstrap from the old letsencrypt-auto, this time installing python3.4
+"$LE_AUTO_PY_34" --no-self-upgrade -n --version >/dev/null 2>/dev/null
 
-# ensure python 3 isn't installed
-if python3 --version 2> /dev/null; then
-  echo "letsencrypt-auto installed Python3 even though Python2.7 is present."
+# ensure python 3.4 is installed
+if ! python3.4 --version >/dev/null 2>/dev/null; then
+  echo "ERROR: letsencrypt-auto failed to install Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present."
   exit 1
 fi
-)
 
-echo "PASSED: Did not upgrade to Python3 when Python2.7 is present."
+echo "PASSED: Successfully upgraded to Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present."
 
-# ensure python2.7 isn't available
-if python2.7 --version 2> /dev/null; then
-  echo "Python2.7 is still available."
-  exit 1
-fi
+# As "certbot-auto" (so without implicit --non-interactive flag set), check that the script
+# refuses to install SCL Python 3.6 when run in a non interactive shell (simulated here
+# using | tee /dev/null) if --non-interactive flag is not provided.
+cp "$LE_AUTO" /tmp/certbot-auto
+# NB: Readline has an issue on all Python versions for CentOS 6, making `certbot --version`
+# output an unprintable ASCII character on a new line at the end.
+# So we take the second last line of the output.
+version=$(/tmp/certbot-auto --version 2>/dev/null | tee /dev/null | tail -2 | head -1)
 
-# Skip self upgrade due to Python 3 not being available.
-if ! "$LE_AUTO" 2>&1 | grep -q "WARNING: couldn't find Python"; then
-  echo "Python upgrade failure warning not printed!"
+if [ "$version" != "$INITIAL_CERTBOT_VERSION_PY34" ]; then
+  echo "ERROR: certbot-auto upgraded certbot in a non-interactive shell with --non-interactive flag not set."
   exit 1
 fi
 
-# bootstrap, this time installing python3
-"$LE_AUTO" --no-self-upgrade -n --version > /dev/null 2> /dev/null
+echo "PASSED: certbot-auto did not upgrade certbot in a non-interactive shell with --non-interactive flag not set."
 
-# ensure python 3 is installed
-if ! python3 --version > /dev/null; then
-  echo "letsencrypt-auto failed to install Python3 when only Python2.6 is present."
+if [ -f /opt/rh/rh-python36/enable ]; then
+  echo "ERROR: certbot-auto installed Python3.6 in a non-interactive shell with --non-interactive flag not set."
   exit 1
 fi
 
-echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present."
-echo ""
+echo "PASSED: certbot-auto did not install Python3.6 in a non-interactive shell with --non-interactive flag not set."
 
-export VENV_PATH=$(mktemp -d)
-"$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1
-if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1)" != 3 ]; then
-  echo "Python 3 wasn't used with --no-bootstrap!"
-  exit 1
-fi
-unset VENV_PATH
+# now bootstrap from current letsencrypt-auto, that will install python3.6 from SCL
+"$LE_AUTO" --no-self-upgrade -n --version >/dev/null 2>/dev/null
+
+# Following test is exectued in a subshell, to not leak any environment variable
+(
+  # enable SCL rh-python36
+  . /opt/rh/rh-python36/enable
+
+  # ensure python 3.6 is installed
+  if ! python3.6 --version >/dev/null 2>/dev/null; then
+    echo "ERROR: letsencrypt-auto failed to install Python3.6 using current letsencrypt-auto when only Python2.6/Python3.4 are present."
+    exit 1
+  fi
+
+  echo "PASSED: Successfully upgraded to Python3.6 using current letsencrypt-auto when only Python2.6/Python3.4 are present."
+)
+
+# Following test is executed in a subshell, to not leak any environment variable
+(
+  export VENV_PATH=$(mktemp -d)
+  "$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1
+  if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1-2)" != "3.6" ]; then
+    echo "ERROR: Python 3.6 wasn't used with --no-bootstrap!"
+    exit 1
+  fi
+)
+
+# Following test is exectued in a subshell, to not leak any environment variable
+(
+  # enable SCL rh-python36
+  . /opt/rh/rh-python36/enable
+
+  # ensure everything works fine with certbot-auto bootstrap when python 3.6 is already enabled
+  export VENV_PATH=$(mktemp -d)
+  if ! "$LE_AUTO" --no-self-upgrade -n --version >/dev/null 2>/dev/null; then
+    echo "ERROR: Certbot-auto broke when Python 3.6 SCL is already enabled."
+    exit 1
+  fi
+)
 
 # we're going to modify env variables, so do this in a subshell
 (
-# ensure CentOS6 32bits is not supported anymore, and so certbot
-# is not upgraded nor reinstalled.
-export UNAME_FAKE_32BITS=true
-set -o pipefail
-if ! "$LE_AUTO" --version 2>&1 | grep -q "Certbot will no longer receive updates."; then
-  echo "On CentOS 6 32 bits, certbot-auto failed or upgraded installed certbot instance."
-  exit 1
-fi
-set +o pipefail
-if ! "$LE_AUTO" --install-only 2>&1 | grep -q "Certbot cannot be installed."; then
-  echo "On CentOS 6 32 bits, certbot-auto installed certbot again."
-  exit 1
-fi
+  # ensure CentOS6 32bits is not supported anymore, and so certbot
+  # is not upgraded nor reinstalled.
+  export UNAME_FAKE_32BITS=true
+  OUTPUT=$("$LE_AUTO" --version 2>&1)
+  if ! echo "$OUTPUT" | grep -q "Certbot will no longer receive updates."; then
+    echo "ERROR: certbot-auto failed to run or upgraded pre-existing Certbot instance on 32-bit CentOS 6."
+    exit 1
+  fi
+  if ! "$LE_AUTO" --install-only 2>&1 | grep -q "Certbot cannot be installed."; then
+    echo "ERROR: certbot-auto reinstalled Certbot on 32-bit CentOS 6."
+    exit 1
+  fi
 )
 
 # we're going to modify env variables, so do this in a subshell
 (
-# Prepare a certbot installation in the old venv path
-rm -rf /opt/eff.org
-VENV_PATH=~/.local/share/letsencrypt "$LE_AUTO" --install-only > /dev/null 2> /dev/null
-# fake 32 bits mode
-export UNAME_FAKE_32BITS=true
-set -o pipefail
-if ! "$LE_AUTO" --version 2>&1 | grep -q "Certbot will no longer receive updates."; then
-  echo "On CentOS 6 32 bits, certbot-auto failed or upgraded installed certbot in the old venv path."
-  exit 1
-fi
-set +o pipefail
+  # Prepare a certbot installation in the old venv path
+  rm -rf /opt/eff.org
+  VENV_PATH=~/.local/share/letsencrypt "$LE_AUTO" --install-only > /dev/null 2> /dev/null
+  # fake 32 bits mode
+  export UNAME_FAKE_32BITS=true
+  OUTPUT=$("$LE_AUTO" --version 2>&1)
+  if ! echo "$OUTPUT" | grep -q "Certbot will no longer receive updates."; then
+    echo "ERROR: certbot-auto failed to run or upgraded pre-existing Certbot instance in the old venv path on 32-bit CentOS 6."
+    exit 1
+  fi
 )
 
-echo "PASSED: On CentOS 6 32 bits, certbot-auto refused to install/upgrade certbot."
+echo "PASSED: certbot-auto refused to install/upgrade certbot on 32-bit CentOS 6."
 
 # test using python3
 pytest -v -s certbot/letsencrypt-auto-source/tests
diff --git a/letsencrypt-auto-source/tests/oraclelinux6_tests.sh b/letsencrypt-auto-source/tests/oraclelinux6_tests.sh
new file mode 100644
--- /dev/null
+++ b/letsencrypt-auto-source/tests/oraclelinux6_tests.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+set -eo pipefail
+# Start by making sure your system is up-to-date:
+yum update -y >/dev/null
+
+LE_AUTO_PY_34="certbot/letsencrypt-auto-source/letsencrypt-auto_py_34"
+LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto"
+
+# Apply installation instructions from official documentation:
+# https://certbot.eff.org/lets-encrypt/centosrhel6-other
+cp "$LE_AUTO" /usr/local/bin/certbot-auto
+chown root /usr/local/bin/certbot-auto
+chmod 0755 /usr/local/bin/certbot-auto
+LE_AUTO=/usr/local/bin/certbot-auto
+
+# Last version of certbot-auto that was bootstraping Python 3.4 for CentOS 6 users
+INITIAL_CERTBOT_VERSION_PY34="certbot 0.38.0"
+
+# Check bootstrap from current certbot-auto will fail, because SCL is not enabled.
+set +o pipefail
+if ! "$LE_AUTO" -n 2>&1 | grep -q "Enable the SCL repository and try running Certbot again."; then
+  echo "ERROR: Bootstrap was not aborted although SCL was not installed!"
+  exit 1
+fi
+set -o pipefail
+
+echo "PASSED: Bootstrap was aborted since SCL was not installed."
+
+# Bootstrap from the old letsencrypt-auto, Python 3.4 will be installed from EPEL.
+"$LE_AUTO_PY_34" --no-self-upgrade -n --install-only >/dev/null 2>/dev/null
+
+# Ensure Python 3.4 is installed
+if ! command -v python3.4 &>/dev/null; then
+  echo "ERROR: old letsencrypt-auto failed to install Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present."
+  exit 1
+fi
+
+echo "PASSED: Bootstrap from old letsencrypt-auto succeeded and installed Python 3.4"
+
+# Expect certbot-auto to skip rebootstrapping with a warning since SCL is not installed.
+if ! "$LE_AUTO" --non-interactive --version 2>&1 | grep -q "This requires manual user intervention"; then
+  echo "FAILED: Script certbot-auto did not print a warning about needing manual intervention!"
+  exit 1
+fi
+
+echo "PASSED: Script certbot-auto did not rebootstrap."
+
+# NB: Readline has an issue on all Python versions for OL 6, making `certbot --version`
+# output an unprintable ASCII character on a new line at the end.
+# So we take the second last line of the output.
+version=$($LE_AUTO --version 2>/dev/null | tail -2 | head -1)
+
+if [ "$version" != "$INITIAL_CERTBOT_VERSION_PY34" ]; then
+  echo "ERROR: Script certbot-auto upgraded certbot in a non-interactive shell while SCL was not enabled."
+  exit 1
+fi
+
+echo "PASSED: Script certbot-auto did not upgrade certbot but started it successfully while SCL was not enabled."
+
+# Enable SCL
+yum install -y oracle-softwarecollection-release-el6 >/dev/null
+
+# Expect certbot-auto to bootstrap successfully since SCL is available.
+"$LE_AUTO" -n --version &>/dev/null
+
+if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1-2)" != "3.6" ]; then
+  echo "ERROR: Script certbot-auto failed to bootstrap and install Python 3.6 while SCL is available."
+  exit 1
+fi
+
+if ! /opt/eff.org/certbot/venv/bin/certbot --version > /dev/null 2> /dev/null; then
+  echo "ERROR: Script certbot-auto did not install certbot correctly while SCL is enabled."
+  exit 1
+fi
+
+echo "PASSED: Script certbot-auto correctly bootstraped Certbot using rh-python36 when SCL is available."
+
+# Expect certbot-auto will be totally silent now that everything has been correctly boostraped.
+OUTPUT_LEN=$("$LE_AUTO" --install-only --no-self-upgrade --quiet 2>&1 | wc -c)
+if [ "$OUTPUT_LEN" != 0 ]; then
+    echo certbot-auto produced unexpected output!
+    exit 1
+fi
+
+echo "PASSED: Script certbot-auto did not print anything in quiet mode."
diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh
--- a/tests/letstest/scripts/test_leauto_upgrades.sh
+++ b/tests/letstest/scripts/test_leauto_upgrades.sh
@@ -117,6 +117,8 @@ if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
 fi
 
 if [ "$RUN_RHEL6_TESTS" = 1 ]; then
+    # Add the SCL python release to PATH in order to resolve python3 command
+    PATH="/opt/rh/rh-python36/root/usr/bin:$PATH"
     if ! command -v python3; then
         echo "Python3 wasn't properly installed"
         exit 1
diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh
--- a/tests/letstest/scripts/test_sdists.sh
+++ b/tests/letstest/scripts/test_sdists.sh
@@ -1,8 +1,21 @@
 #!/bin/sh -xe
 
 cd letsencrypt
+
+# If we're on a RHEL 6 based system, we can be confident Python is already
+# installed because the package manager is written in Python.
+if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then
+    # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment
+    RUN_RHEL6_TESTS=1
+fi
+
 letsencrypt-auto-source/letsencrypt-auto --install-only -n --debug
 
+if [ "$RUN_RHEL6_TESTS" = 1 ]; then
+  # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap
+  PATH="/opt/rh/rh-python36/root/usr/bin:$PATH"
+fi
+
 PLUGINS="certbot-apache certbot-nginx"
 PYTHON_MAJOR_VERSION=$(/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1)
 TEMP_DIR=$(mktemp -d)