Repository: certbot/certbot

Issue:
Bug running unit tests in parallel on Python 2.6
I filed this at nose-devs/nose#1041 but the project seems pretty inactive. We should document this behavior of our tests, fix the problem, or stop running tests in parallel on Python 2.6.


Discussion:

Resolving patch:
diff --git a/acme/setup.py b/acme/setup.py
--- a/acme/setup.py
+++ b/acme/setup.py
@@ -31,7 +31,8 @@
     ])
 
 dev_extras = [
-    'nose',
+    'pytest',
+    'pytest-xdist',
     'tox',
 ]
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,9 @@ def read_file(filename, encoding='utf8'):
     'astroid==1.3.5',
     'coverage',
     'ipdb',
-    'nose',
+    'pytest',
+    'pytest-cov',
+    'pytest-xdist',
     'pylint==1.4.2',  # upstream #248
     'tox',
     'twine',


Test patch:
diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py
--- a/acme/acme/crypto_util_test.py
+++ b/acme/acme/crypto_util_test.py
@@ -18,7 +18,6 @@
 class SSLSocketAndProbeSNITest(unittest.TestCase):
     """Tests for acme.crypto_util.SSLSocket/probe_sni."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.cert = test_util.load_comparable_cert('rsa2048_cert.pem')
@@ -69,7 +68,6 @@ def test_probe_not_recognized_name(self):
 class PyOpenSSLCertOrReqSANTest(unittest.TestCase):
     """Test for acme.crypto_util._pyopenssl_cert_or_req_san."""
 
-    _multiprocess_can_split_ = True
 
     @classmethod
     def _call(cls, loader, name):
@@ -140,7 +138,6 @@ def test_critical_san(self):
 class RandomSnTest(unittest.TestCase):
     """Test for random certificate serial numbers."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.cert_count = 5
diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py
--- a/acme/acme/messages_test.py
+++ b/acme/acme/messages_test.py
@@ -71,6 +71,12 @@ def test_with_code(self):
         self.assertTrue(is_acme_error(Error.with_code('badCSR')))
         self.assertRaises(ValueError, Error.with_code, 'not an ACME error code')
 
+    def test_str(self):
+        self.assertEqual(
+            str(self.error),
+            u"{0.typ} :: {0.description} :: {0.detail} :: {0.title}"
+            .format(self.error))
+
 
 class ConstantTest(unittest.TestCase):
     """Tests for acme.messages._Constant."""
diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py
--- a/acme/acme/standalone_test.py
+++ b/acme/acme/standalone_test.py
@@ -23,7 +23,6 @@
 class TLSServerTest(unittest.TestCase):
     """Tests for acme.standalone.TLSServer."""
 
-    _multiprocess_can_split_ = True
 
     def test_bind(self):  # pylint: disable=no-self-use
         from acme.standalone import TLSServer
@@ -42,7 +41,6 @@ def test_ipv6(self):
 class TLSSNI01ServerTest(unittest.TestCase):
     """Test for acme.standalone.TLSSNI01Server."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.certs = {b'localhost': (
@@ -70,7 +68,6 @@ def test_it(self):
 class HTTP01ServerTest(unittest.TestCase):
     """Tests for acme.standalone.HTTP01Server."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.account_key = jose.JWK.load(
@@ -124,7 +121,6 @@ def test_http01_not_found(self):
 class BaseDualNetworkedServersTest(unittest.TestCase):
     """Test for acme.standalone.BaseDualNetworkedServers."""
 
-    _multiprocess_can_split_ = True
 
     class SingleProtocolServer(socketserver.TCPServer):
         """Server that only serves on a single protocol. FreeBSD has this behavior for AF_INET6."""
@@ -174,7 +170,6 @@ def test_ports_equal(self):
 class TLSSNI01DualNetworkedServersTest(unittest.TestCase):
     """Test for acme.standalone.TLSSNI01DualNetworkedServers."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.certs = {b'localhost': (
@@ -202,7 +197,6 @@ def test_connect(self):
 class HTTP01DualNetworkedServersTest(unittest.TestCase):
     """Tests for acme.standalone.HTTP01DualNetworkedServers."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.account_key = jose.JWK.load(
@@ -254,7 +248,6 @@ def test_http01_not_found(self):
 class TestSimpleTLSSNI01Server(unittest.TestCase):
     """Tests for acme.standalone.simple_tls_sni_01_server."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         # mirror ../examples/standalone
diff --git a/certbot-apache/certbot_apache/tests/augeas_configurator_test.py b/certbot-apache/certbot_apache/tests/augeas_configurator_test.py
--- a/certbot-apache/certbot_apache/tests/augeas_configurator_test.py
+++ b/certbot-apache/certbot_apache/tests/augeas_configurator_test.py
@@ -13,7 +13,6 @@
 class AugeasConfiguratorTest(util.ApacheTest):
     """Test for Augeas Configurator base class."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):  # pylint: disable=arguments-differ
         super(AugeasConfiguratorTest, self).setUp()
diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py
--- a/certbot-apache/certbot_apache/tests/configurator_test.py
+++ b/certbot-apache/certbot_apache/tests/configurator_test.py
@@ -30,7 +30,6 @@
 class MultipleVhostsTest(util.ApacheTest):
     """Test two standard well-configured HTTP vhosts."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):  # pylint: disable=arguments-differ
         super(MultipleVhostsTest, self).setUp()
@@ -1369,7 +1368,6 @@ def test_aug_version(self):
 class AugeasVhostsTest(util.ApacheTest):
     """Test vhosts with illegal names dependent on augeas version."""
     # pylint: disable=protected-access
-    _multiprocess_can_split_ = True
 
     def setUp(self):  # pylint: disable=arguments-differ
         td = "debian_apache_2_4/augeas_vhosts"
diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py
--- a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py
+++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py
@@ -5,7 +5,6 @@
 
 import digitalocean
 import mock
-import six
 
 from certbot import errors
 from certbot.plugins import dns_test_common
@@ -134,8 +133,8 @@ def test_del_txt_record(self):
 
         correct_record_mock.destroy.assert_called()
 
-        six.assertCountEqual(self, first_record_mock.destroy.call_args_list, [])
-        six.assertCountEqual(self, last_record_mock.destroy.call_args_list, [])
+        self.assertFalse(first_record_mock.destroy.call_args_list)
+        self.assertFalse(last_record_mock.destroy.call_args_list)
 
     def test_del_txt_record_error_finding_domain(self):
         self.manager.get_all_domains.side_effect = API_ERROR
diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py
--- a/certbot-nginx/certbot_nginx/tests/configurator_test.py
+++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py
@@ -24,7 +24,6 @@
 class NginxConfiguratorTest(util.NginxTest):
     """Test a semi complex vhost configuration."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         super(NginxConfiguratorTest, self).setUp()
diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py
--- a/certbot/tests/cli_test.py
+++ b/certbot/tests/cli_test.py
@@ -26,7 +26,6 @@
 class TestReadFile(TempDirTestCase):
     '''Test cli.read_file'''
 
-    _multiprocess_can_split_ = True
 
     def test_read_file(self):
         rel_test_path = os.path.relpath(os.path.join(self.tempdir, 'foo'))
@@ -46,7 +45,6 @@ def test_read_file(self):
 class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
     '''Test the cli args entrypoint'''
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         reload_module(cli)
@@ -418,7 +416,6 @@ def test_no_directory_hooks_unset(self):
 class DefaultTest(unittest.TestCase):
     """Tests for certbot.cli._Default."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         # pylint: disable=protected-access
@@ -439,7 +436,6 @@ def test_hash(self):
 class SetByCliTest(unittest.TestCase):
     """Tests for certbot.set_by_cli and related functions."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         reload_module(cli)
diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py
--- a/certbot/tests/ocsp_test.py
+++ b/certbot/tests/ocsp_test.py
@@ -13,7 +13,6 @@
 
 class OCSPTest(unittest.TestCase):
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         from certbot import ocsp
diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py
--- a/certbot/tests/storage_test.py
+++ b/certbot/tests/storage_test.py
@@ -43,7 +43,6 @@ class BaseRenewableCertTest(test_util.ConfigTestCase):
     your test.  Check :class:`.cli_test.DuplicateCertTest` for an example.
 
     """
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         from certbot import storage
diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py
--- a/letsencrypt-auto-source/tests/auto_test.py
+++ b/letsencrypt-auto-source/tests/auto_test.py
@@ -17,10 +17,10 @@
 from threading import Thread
 from unittest import TestCase
 
-from nose.tools import eq_, nottest, ok_
+from pytest import mark
 
 
-@nottest
+@mark.skip
 def tests_dir():
     """Return a path to the "tests" directory."""
     return dirname(abspath(__file__))
@@ -279,8 +279,8 @@ def test_successes(self):
                 # installed, and pip hashes verify:
                 install_le_auto(build_le_auto(version='50.0.0'), le_auto_path)
                 out, err = run_letsencrypt_auto()
-                ok_(re.match(r'letsencrypt \d+\.\d+\.\d+',
-                             err.strip().splitlines()[-1]))
+                self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+',
+                                err.strip().splitlines()[-1]))
                 # Make a few assertions to test the validity of the next tests:
                 self.assertTrue('Upgrading certbot-auto ' in out)
                 self.assertTrue('Creating virtual environment...' in out)
@@ -327,7 +327,7 @@ def test_openssl_failure(self):
                 try:
                     out, err = run_le_auto(le_auto_path, venv_dir, base_url)
                 except CalledProcessError as exc:
-                    eq_(exc.returncode, 1)
+                    self.assertEqual(exc.returncode, 1)
                     self.assertTrue("Couldn't verify signature of downloaded "
                                     "certbot-auto." in exc.output)
                 else:
@@ -348,10 +348,11 @@ def test_pip_failure(self):
                 try:
                     out, err = run_le_auto(le_auto_path, venv_dir, base_url)
                 except CalledProcessError as exc:
-                    eq_(exc.returncode, 1)
+                    self.assertEqual(exc.returncode, 1)
                     self.assertTrue("THESE PACKAGES DO NOT MATCH THE HASHES "
                                     "FROM THE REQUIREMENTS FILE" in exc.output)
-                    ok_(not exists(venv_dir),
+                    self.assertFalse(
+                        exists(venv_dir),
                         msg="The virtualenv was left around, even though "
                             "installation didn't succeed. We shouldn't do "
                             "this, as it foils our detection of whether we "
diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh
--- a/tests/letstest/scripts/test_tests.sh
+++ b/tests/letstest/scripts/test_tests.sh
@@ -10,9 +10,9 @@ LE_AUTO_SUDO="" VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --no-boots
 
 # change to an empty directory to ensure CWD doesn't affect tests
 cd $(mktemp -d)
-pip install nose
+pip install pytest==3.2.5
 
 for module in $MODULES ; do
     echo testing $module
-    nosetests -v $module
+    pytest -v --pyargs $module
 done
diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh
--- a/tools/install_and_test.sh
+++ b/tools/install_and_test.sh
@@ -16,6 +16,10 @@ for requirement in "$@" ; do
   pkg=$(echo $requirement | cut -f1 -d\[)  # remove any extras such as [dev]
   if [ $pkg = "." ]; then
     pkg="certbot"
+  else
+    # Work around a bug in pytest/importlib for the deprecated Python 3.3.
+    # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333.
+    pkg=$(echo "$pkg" | tr - _)
   fi
-  nosetests -v $pkg --processes=-1 --process-timeout=100
+  pytest --numprocesses auto --quiet --pyargs $pkg
 done