Repository: certbot/certbot

Issue:
Prevent regressions of problems like #4044
The cause of #4044 is that although `setUpClass` wasn't added until Python 2.7, [nose manually runs this method for us](https://github.com/nose-devs/nose/blob/master/nose/suite.py#L308). We should make sure our tests catch problems like this in the future.

Options:

1. Stop using `nose` and rely either on standard Python scripts for running tests or use another program to run our tests.
2. Use another test (perhaps like my hacky script [here](https://github.com/certbot/certbot/pull/4043#issuecomment-272350209)) and make sure it runs regularly on a Python 2.6 system.
3. ???


Discussion:

Resolving patch:
diff --git a/letsencrypt-auto-source/build.py b/letsencrypt-auto-source/build.py
--- a/letsencrypt-auto-source/build.py
+++ b/letsencrypt-auto-source/build.py
@@ -8,24 +8,11 @@
 """
 from os.path import abspath, dirname, join
 import re
-from sys import argv
 
+from version import certbot_version, file_contents
 
-DIR = dirname(abspath(__file__))
-
-
-def certbot_version(build_script_dir):
-    """Return the version number stamped in certbot/__init__.py."""
-    return re.search('''^__version__ = ['"](.+)['"].*''',
-                     file_contents(join(dirname(build_script_dir),
-                                        'certbot',
-                                        '__init__.py')),
-                     re.M).group(1)
 
-
-def file_contents(path):
-    with open(path) as file:
-        return file.read()
+DIR = dirname(abspath(__file__))
 
 
 def build(version=None, requirements=None):
diff --git a/letsencrypt-auto-source/version.py b/letsencrypt-auto-source/version.py
new file mode 100755
--- /dev/null
+++ b/letsencrypt-auto-source/version.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+"""Get the current Certbot version number.
+
+Provides simple utilities for determining the Certbot version number and
+building letsencrypt-auto.
+
+"""
+from __future__ import print_function
+from os.path import abspath, dirname, join
+import re
+
+
+def certbot_version(build_script_dir):
+    """Return the version number stamped in certbot/__init__.py."""
+    return re.search('''^__version__ = ['"](.+)['"].*''',
+                     file_contents(join(dirname(build_script_dir),
+                                        'certbot',
+                                        '__init__.py')),
+                     re.M).group(1)
+
+
+def file_contents(path):
+    with open(path) as file:
+        return file.read()
+
+
+if __name__ == '__main__':
+    print(certbot_version(dirname(abspath(__file__))))


Test patch:
diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh
--- a/tests/letstest/scripts/test_leauto_upgrades.sh
+++ b/tests/letstest/scripts/test_leauto_upgrades.sh
@@ -29,7 +29,8 @@ unset PIP_INDEX_URL
 export PIP_EXTRA_INDEX_URL="$SAVE"
 
 git checkout -f "$BRANCH"
-if ! ./letsencrypt-auto -v --debug --version | grep 0.9.0 ; then
+EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2)
+if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade | grep $EXPECTED_VERSION ; then
     echo upgrade appeared to fail
     exit 1
 fi
diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh
--- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh
+++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh
@@ -23,7 +23,7 @@ cd tests/letstest/testdata/
 
 # 2. refer to the config with the same level of relativity that it itself
 # contains :/
-OUT=`letsencrypt-auto certificates --config-dir sample-config -v`
+OUT=`letsencrypt-auto certificates --config-dir sample-config -v --no-self-upgrade`
 TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l`
 REVOKED=`echo "$OUT" | grep REVOKED | wc -l`
 
diff --git a/tests/letstest/scripts/test_ocsp_experimental.sh b/tests/letstest/scripts/test_ocsp_experimental.sh
deleted file mode 100755
--- a/tests/letstest/scripts/test_ocsp_experimental.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash -x
-
-# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
-
-# with curl, instance metadata available from EC2 metadata service:
-#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
-#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
-#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
-
-cd letsencrypt
-export PATH="$PWD/letsencrypt-auto-source:$PATH"
-letsencrypt-auto-source/letsencrypt-auto --os-packages-only --debug --version
-tools/venv.sh
-sudo venv/bin/certbot certonly --no-self-upgrade -v --standalone --debug \
-                   --text --agree-dev-preview --agree-tos \
-                   --renew-by-default --redirect \
-                   --register-unsafely-without-email \
-                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
-
-# we have to jump through some hoops to cope with relative paths in renewal
-# conf files ...
-# 1. be in the right directory
-cd tests/letstest/testdata/
-
-# 2. refer to the config with the same level of relativity that it itself
-# contains :/
-OUT=`sudo ../../../venv/bin/certbot certificates  -v --config-dir sample-config`
-TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l`
-REVOKED=`echo "$OUT" | grep REVOKED | wc -l`
-
-if [ "$TEST_CERTS" != 2 ] ; then
-    echo "Did not find two test certs as expected ($TEST_CERTS)"
-    exit 1
-fi
-
-if [ "$REVOKED" != 1 ] ; then
-    echo "Did not find one revoked cert as expected ($REVOKED)"
-    exit 1
-fi
diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh
new file mode 100755
--- /dev/null
+++ b/tests/letstest/scripts/test_sdists.sh
@@ -0,0 +1,36 @@
+#!/bin/sh -xe
+
+cd letsencrypt
+./certbot-auto --os-packages-only -n --debug
+
+PLUGINS="certbot-apache certbot-nginx"
+PYTHON=$(command -v python2.7 || command -v python27 || command -v python2 || command -v python)
+TEMP_DIR=$(mktemp -d)
+VERSION=$(letsencrypt-auto-source/version.py)
+
+# setup venv
+virtualenv --no-site-packages -p $PYTHON --setuptools venv
+. ./venv/bin/activate
+pip install -U pip
+pip install -U setuptools
+
+# build sdists
+for pkg_dir in acme . $PLUGINS; do
+    cd $pkg_dir
+    python setup.py clean
+    rm -rf build dist
+    python setup.py sdist
+    mv dist/* $TEMP_DIR
+    cd -
+done
+
+# test sdists
+cd $TEMP_DIR
+for pkg in acme certbot $PLUGINS; do
+    tar -xvf "$pkg-$VERSION.tar.gz"
+    cd "$pkg-$VERSION"
+    python setup.py build
+    python setup.py test
+    python setup.py install
+    cd -
+done