Repository: certbot/certbot

Issue:
Certbot struggles with hundreds of thousands certificates
See https://community.letsencrypt.org/t/performance-issues-when-creating-renewing-certificates/26341.

Putting this in 0.11.0 for now, but we can move it as necessary.


Discussion:
One potentially easy fix here is to use `certbot.storage.renewal_filename_for_lineagename` to find the lineage matching `--cert-name` rather than iterating through `/etc/letsencrypt/renewal`. The fix/workaround would then be to include a `--cert-name` flag.

We'd need to carefully verify we don't search `/etc/letsencrypt/renewal` anywhere else though when creating/modifying a cert with `certonly`.
cc @ohemorange
My hypothesis in that thread is that the existing command line flag `--duplicate` will succeed in skipping this check and making things faster. (Though it's not clear that this is the best solution or that people should be required to do this.)

Resolving patch:
diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py
--- a/certbot/cert_manager.py
+++ b/certbot/cert_manager.py
@@ -95,27 +95,23 @@ def delete(config):
 # Public Helpers
 ###################
 
-def lineage_for_certname(config, certname):
+def lineage_for_certname(cli_config, certname):
     """Find a lineage object with name certname."""
-    def update_cert_for_name_match(candidate_lineage, rv):
-        """Return cert if it has name certname, else return rv
-        """
-        matching_lineage_name_cert = rv
-        if candidate_lineage.lineagename == certname:
-            matching_lineage_name_cert = candidate_lineage
-        return matching_lineage_name_cert
-    return _search_lineages(config, update_cert_for_name_match, None)
+    configs_dir = cli_config.renewal_configs_dir
+    # Verify the directory is there
+    util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid())
+    renewal_file = storage.renewal_file_for_certname(cli_config, certname)
+    try:
+        return storage.RenewableCert(renewal_file, cli_config)
+    except (errors.CertStorageError, IOError):
+        logger.debug("Renewal conf file %s is broken.", renewal_file)
+        logger.debug("Traceback was:\n%s", traceback.format_exc())
+        return None
 
 def domains_for_certname(config, certname):
     """Find the domains in the cert with name certname."""
-    def update_domains_for_name_match(candidate_lineage, rv):
-        """Return domains if certname matches, else return rv
-        """
-        matching_domains = rv
-        if candidate_lineage.lineagename == certname:
-            matching_domains = candidate_lineage.names()
-        return matching_domains
-    return _search_lineages(config, update_domains_for_name_match, None)
+    lineage = lineage_for_certname(config, certname)
+    return lineage.names() if lineage else None
 
 def find_duplicative_certs(config, domains):
     """Find existing certs that duplicate the request."""


Test patch:
diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py
--- a/certbot/tests/cert_manager_test.py
+++ b/certbot/tests/cert_manager_test.py
@@ -268,11 +268,11 @@ class LineageForCertnameTest(BaseCertManagerTest):
     """Tests for certbot.cert_manager.lineage_for_certname"""
 
     @mock.patch('certbot.util.make_or_verify_dir')
-    @mock.patch('certbot.storage.renewal_conf_files')
+    @mock.patch('certbot.storage.renewal_file_for_certname')
     @mock.patch('certbot.storage.RenewableCert')
-    def test_found_match(self, mock_renewable_cert, mock_renewal_conf_files,
+    def test_found_match(self, mock_renewable_cert, mock_renewal_conf_file,
         mock_make_or_verify_dir):
-        mock_renewal_conf_files.return_value = ["somefile.conf"]
+        mock_renewal_conf_file.return_value = "somefile.conf"
         mock_match = mock.Mock(lineagename="example.com")
         mock_renewable_cert.return_value = mock_match
         from certbot import cert_manager
@@ -281,13 +281,10 @@ def test_found_match(self, mock_renewable_cert, mock_renewal_conf_files,
         self.assertTrue(mock_make_or_verify_dir.called)
 
     @mock.patch('certbot.util.make_or_verify_dir')
-    @mock.patch('certbot.storage.renewal_conf_files')
-    @mock.patch('certbot.storage.RenewableCert')
-    def test_no_match(self, mock_renewable_cert, mock_renewal_conf_files,
+    @mock.patch('certbot.storage.renewal_file_for_certname')
+    def test_no_match(self, mock_renewal_conf_file,
         mock_make_or_verify_dir):
-        mock_renewal_conf_files.return_value = ["somefile.conf"]
-        mock_match = mock.Mock(lineagename="other.com")
-        mock_renewable_cert.return_value = mock_match
+        mock_renewal_conf_file.return_value = "other.com.conf"
         from certbot import cert_manager
         self.assertEqual(cert_manager.lineage_for_certname(self.cli_config, "example.com"),
             None)
@@ -298,11 +295,11 @@ class DomainsForCertnameTest(BaseCertManagerTest):
     """Tests for certbot.cert_manager.domains_for_certname"""
 
     @mock.patch('certbot.util.make_or_verify_dir')
-    @mock.patch('certbot.storage.renewal_conf_files')
+    @mock.patch('certbot.storage.renewal_file_for_certname')
     @mock.patch('certbot.storage.RenewableCert')
-    def test_found_match(self, mock_renewable_cert, mock_renewal_conf_files,
+    def test_found_match(self, mock_renewable_cert, mock_renewal_conf_file,
         mock_make_or_verify_dir):
-        mock_renewal_conf_files.return_value = ["somefile.conf"]
+        mock_renewal_conf_file.return_value = "somefile.conf"
         mock_match = mock.Mock(lineagename="example.com")
         domains = ["example.com", "example.org"]
         mock_match.names.return_value = domains
@@ -313,15 +310,10 @@ def test_found_match(self, mock_renewable_cert, mock_renewal_conf_files,
         self.assertTrue(mock_make_or_verify_dir.called)
 
     @mock.patch('certbot.util.make_or_verify_dir')
-    @mock.patch('certbot.storage.renewal_conf_files')
-    @mock.patch('certbot.storage.RenewableCert')
-    def test_no_match(self, mock_renewable_cert, mock_renewal_conf_files,
+    @mock.patch('certbot.storage.renewal_file_for_certname')
+    def test_no_match(self, mock_renewal_conf_file,
         mock_make_or_verify_dir):
-        mock_renewal_conf_files.return_value = ["somefile.conf"]
-        mock_match = mock.Mock(lineagename="example.com")
-        domains = ["example.com", "example.org"]
-        mock_match.names.return_value = domains
-        mock_renewable_cert.return_value = mock_match
+        mock_renewal_conf_file.return_value = "somefile.conf"
         from certbot import cert_manager
         self.assertEqual(cert_manager.domains_for_certname(self.cli_config, "other.com"),
             None)