Repository: certbot/certbot

Issue:
No logging in letsencrypt-renewer
https://github.com/letsencrypt/letsencrypt/pull/895#issuecomment-145390510



Discussion:
Related to this, there is no logging in `storage.py` either, which should perhaps be made into it's own issue.


Resolving patch:
diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py
--- a/letsencrypt/cli.py
+++ b/letsencrypt/cli.py
@@ -841,9 +841,23 @@ def _plugins_parsing(helpful, plugins):
     helpful.add_plugin_args(plugins)
 
 
-def _setup_logging(args):
-    level = -args.verbose_count * 10
-    fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s"
+def setup_log_file_handler(args, logfile, fmt):
+    """Setup file debug logging."""
+    log_file_path = os.path.join(args.logs_dir, logfile)
+    handler = logging.handlers.RotatingFileHandler(
+        log_file_path, maxBytes=2 ** 20, backupCount=10)
+    # rotate on each invocation, rollover only possible when maxBytes
+    # is nonzero and backupCount is nonzero, so we set maxBytes as big
+    # as possible not to overrun in single CLI invocation (1MB).
+    handler.doRollover()  # TODO: creates empty letsencrypt.log.1 file
+    handler.setLevel(logging.DEBUG)
+    handler_formatter = logging.Formatter(fmt=fmt)
+    handler_formatter.converter = time.gmtime  # don't use localtime
+    handler.setFormatter(handler_formatter)
+    return handler, log_file_path
+
+
+def _cli_log_handler(args, level, fmt):
     if args.text_mode:
         handler = colored_logging.StreamHandler()
         handler.setFormatter(logging.Formatter(fmt))
@@ -852,30 +866,26 @@ def _setup_logging(args):
         # dialog box is small, display as less as possible
         handler.setFormatter(logging.Formatter("%(message)s"))
     handler.setLevel(level)
+    return handler
 
-    # TODO: use fileConfig?
 
-    # unconditionally log to file for debugging purposes
-    # TODO: change before release?
-    log_file_name = os.path.join(args.logs_dir, 'letsencrypt.log')
-    file_handler = logging.handlers.RotatingFileHandler(
-        log_file_name, maxBytes=2 ** 20, backupCount=10)
-    # rotate on each invocation, rollover only possible when maxBytes
-    # is nonzero and backupCount is nonzero, so we set maxBytes as big
-    # as possible not to overrun in single CLI invocation (1MB).
-    file_handler.doRollover()  # TODO: creates empty letsencrypt.log.1 file
-    file_handler.setLevel(logging.DEBUG)
-    file_handler_formatter = logging.Formatter(fmt=fmt)
-    file_handler_formatter.converter = time.gmtime  # don't use localtime
-    file_handler.setFormatter(file_handler_formatter)
+def setup_logging(args, cli_handler_factory, logfile):
+    """Setup logging."""
+    fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s"
+    level = -args.verbose_count * 10
+    file_handler, log_file_path = setup_log_file_handler(
+        args, logfile=logfile, fmt=fmt)
+    cli_handler = cli_handler_factory(args, level, fmt)
+
+    # TODO: use fileConfig?
 
     root_logger = logging.getLogger()
     root_logger.setLevel(logging.DEBUG)  # send all records to handlers
-    root_logger.addHandler(handler)
+    root_logger.addHandler(cli_handler)
     root_logger.addHandler(file_handler)
 
     logger.debug("Root logging level set at %d", level)
-    logger.info("Saving debug log to %s", log_file_name)
+    logger.info("Saving debug log to %s", log_file_path)
 
 
 def _handle_exception(exc_type, exc_value, trace, args):
@@ -944,7 +954,7 @@ def main(cli_args=sys.argv[1:]):
     # private key! #525
     le_util.make_or_verify_dir(
         args.logs_dir, 0o700, os.geteuid(), "--strict-permissions" in cli_args)
-    _setup_logging(args)
+    setup_logging(args, _cli_log_handler, logfile='letsencrypt.log')
 
     # do not log `args`, as it contains sensitive data (e.g. revoke --key)!
     logger.debug("Arguments: %r", cli_args)
diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py
--- a/letsencrypt/renewer.py
+++ b/letsencrypt/renewer.py
@@ -8,6 +8,7 @@
 
 """
 import argparse
+import logging
 import os
 import sys
 
@@ -17,10 +18,12 @@
 
 from letsencrypt import account
 from letsencrypt import configuration
+from letsencrypt import colored_logging
 from letsencrypt import cli
 from letsencrypt import client
 from letsencrypt import crypto_util
 from letsencrypt import errors
+from letsencrypt import le_util
 from letsencrypt import notify
 from letsencrypt import storage
 
@@ -28,6 +31,9 @@
 from letsencrypt.plugins import disco as plugins_disco
 
 
+logger = logging.getLogger(__name__)
+
+
 class _AttrDict(dict):
     """Attribute dictionary.
 
@@ -104,6 +110,12 @@ def renew(cert, old_version):
     #       (where fewer than all names were renewed)
 
 
+def _cli_log_handler(args, level, fmt):  # pylint: disable=unused-argument
+    handler = colored_logging.StreamHandler()
+    handler.setFormatter(logging.Formatter(fmt))
+    return handler
+
+
 def _paths_parser(parser):
     add = parser.add_argument_group("paths").add_argument
     add("--config-dir", default=cli.flag_default("config_dir"),
@@ -119,11 +131,16 @@ def _paths_parser(parser):
 def _create_parser():
     parser = argparse.ArgumentParser()
     #parser.add_argument("--cron", action="store_true", help="Run as cronjob.")
-    # pylint: disable=protected-access
+    parser.add_argument(
+        "-v", "--verbose", dest="verbose_count", action="count",
+        default=cli.flag_default("verbose_count"), help="This flag can be used "
+        "multiple times to incrementally increase the verbosity of output, "
+        "e.g. -vvv.")
+
     return _paths_parser(parser)
 
 
-def main(config=None, args=sys.argv[1:]):
+def main(config=None, cli_args=sys.argv[1:]):
     """Main function for autorenewer script."""
     # TODO: Distinguish automated invocation from manual invocation,
     #       perhaps by looking at sys.argv[0] and inhibiting automated
@@ -133,8 +150,12 @@ def main(config=None, args=sys.argv[1:]):
 
     zope.component.provideUtility(display_util.FileDisplay(sys.stdout))
 
-    cli_config = configuration.RenewerConfiguration(
-        _create_parser().parse_args(args))
+    args = _create_parser().parse_args(cli_args)
+
+    le_util.make_or_verify_dir(args.logs_dir, 0o700, os.geteuid())
+    cli.setup_logging(args, _cli_log_handler, logfile='renewer.log')
+
+    cli_config = configuration.RenewerConfiguration(args)
 
     config = storage.config_with_defaults(config)
     # Now attempt to read the renewer config file and augment or replace


Test patch:
diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py
--- a/letsencrypt/tests/renewer_test.py
+++ b/letsencrypt/tests/renewer_test.py
@@ -44,8 +44,15 @@ def setUp(self):
         self.tempdir = tempfile.mkdtemp()
 
         self.cli_config = configuration.RenewerConfiguration(
-            namespace=mock.MagicMock(config_dir=self.tempdir))
+            namespace=mock.MagicMock(
+                config_dir=self.tempdir,
+                work_dir=self.tempdir,
+                logs_dir=self.tempdir,
+            )
+        )
+
         # TODO: maybe provide RenewerConfiguration.make_dirs?
+        # TODO: main() should create those dirs, c.f. #902
         os.makedirs(os.path.join(self.tempdir, "live", "example.org"))
         os.makedirs(os.path.join(self.tempdir, "archive", "example.org"))
         os.makedirs(os.path.join(self.tempdir, "renewal"))
@@ -62,6 +69,9 @@ def setUp(self):
         self.test_rc = storage.RenewableCert(
             self.config, self.defaults, self.cli_config)
 
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
     def _write_out_ex_kinds(self):
         for kind in ALL_FOUR:
             where = getattr(self.test_rc, kind)
@@ -79,11 +89,6 @@ def _write_out_ex_kinds(self):
 class RenewableCertTests(BaseRenewableCertTest):
     # pylint: disable=too-many-public-methods
     """Tests for letsencrypt.renewer.*."""
-    def setUp(self):
-        super(RenewableCertTests, self).setUp()
-
-    def tearDown(self):
-        shutil.rmtree(self.tempdir)
 
     def test_initialization(self):
         self.assertEqual(self.test_rc.lineagename, "example.org")
@@ -665,11 +670,17 @@ def test_renew(self, mock_c, mock_acc_storage, mock_pd):
         # This should fail because the renewal itself appears to fail
         self.assertFalse(renewer.renew(self.test_rc, 1))
 
+    def _common_cli_args(self):
+        return [
+            "--config-dir", self.cli_config.config_dir,
+            "--work-dir", self.cli_config.work_dir,
+            "--logs-dir", self.cli_config.logs_dir,
+        ]
+
     @mock.patch("letsencrypt.renewer.notify")
     @mock.patch("letsencrypt.storage.RenewableCert")
     @mock.patch("letsencrypt.renewer.renew")
     def test_main(self, mock_renew, mock_rc, mock_notify):
-        """Test for main() function."""
         from letsencrypt import renewer
         mock_rc_instance = mock.MagicMock()
         mock_rc_instance.should_autodeploy.return_value = True
@@ -691,8 +702,7 @@ def test_main(self, mock_renew, mock_rc, mock_notify):
                                "example.com.conf"), "w") as f:
             f.write("cert = cert.pem\nprivkey = privkey.pem\n")
             f.write("chain = chain.pem\nfullchain = fullchain.pem\n")
-        renewer.main(self.defaults, args=[
-            '--config-dir', self.cli_config.config_dir])
+        renewer.main(self.defaults, cli_args=self._common_cli_args())
         self.assertEqual(mock_rc.call_count, 2)
         self.assertEqual(mock_rc_instance.update_all_links_to.call_count, 2)
         self.assertEqual(mock_notify.notify.call_count, 4)
@@ -705,8 +715,7 @@ def test_main(self, mock_renew, mock_rc, mock_notify):
         mock_happy_instance.should_autorenew.return_value = False
         mock_happy_instance.latest_common_version.return_value = 10
         mock_rc.return_value = mock_happy_instance
-        renewer.main(self.defaults, args=[
-            '--config-dir', self.cli_config.config_dir])
+        renewer.main(self.defaults, cli_args=self._common_cli_args())
         self.assertEqual(mock_rc.call_count, 4)
         self.assertEqual(mock_happy_instance.update_all_links_to.call_count, 0)
         self.assertEqual(mock_notify.notify.call_count, 4)
@@ -717,8 +726,7 @@ def test_bad_config_file(self):
         with open(os.path.join(self.cli_config.renewal_configs_dir,
                                "bad.conf"), "w") as f:
             f.write("incomplete = configfile\n")
-        renewer.main(self.defaults, args=[
-            '--config-dir', self.cli_config.config_dir])
+        renewer.main(self.defaults, cli_args=self._common_cli_args())
         # The errors.CertStorageError is caught inside and nothing happens.