Repository: mkdocs/mkdocs

Issue:
Improve CLI tests.
The CLI tests use Mock objects (as they should), but all they do is confirm that the proper function was called once. There is no testing of the CLI options and what values are passed in (either the default values or custom values defined by the user). For example, when working on #997, both @aeslaughter and I missed that the default server was changed (see #1014). Also while working on #1009, I noticed that perhaps the default for `--clean` may not be what we think it is (we intended to change the default in  #997, but I'm not sure we did).

We need tests for all the different options for each command to ensure the values passed from the CLI match expectations/documentation. Basically, we need more tests like the one I added in #1009.

Discussion:

Patch:
diff --git a/mkdocs/__main__.py b/mkdocs/__main__.py
index c9d519fc..016564b3 100644
--- a/mkdocs/__main__.py
+++ b/mkdocs/__main__.py
@@ -67,7 +67,7 @@ def common_options(f):
     return f
 
 
-clean_help = "Remove old files from the site_dir before building"
+clean_help = "Remove old files from the site_dir before building (the default)."
 config_help = "Provide a specific MkDocs config"
 dev_addr_help = ("IP address and port to serve documentation locally (default: "
                  "localhost:8000)")
@@ -103,7 +103,7 @@ def cli():
 @click.option('-s', '--strict', is_flag=True, help=strict_help)
 @click.option('-t', '--theme', type=click.Choice(theme_choices), help=theme_help)
 @click.option('-e', '--theme-dir', type=click.Path(), help=theme_dir_help)
-@click.option('--livereload', 'livereload', flag_value='livereload', help=reload_help)
+@click.option('--livereload', 'livereload', flag_value='livereload', help=reload_help, default=True)
 @click.option('--no-livereload', 'livereload', flag_value='no-livereload', help=no_reload_help)
 @click.option('-d', '--dirtyreload', 'livereload', flag_value='dirty', help=dirty_reload_help)
 @common_options
@@ -125,13 +125,13 @@ def serve_command(dev_addr, config_file, strict, theme, theme_dir, livereload):
             theme_dir=theme_dir,
             livereload=livereload
         )
-    except (exceptions.ConfigurationError, socket.error) as e:
+    except (exceptions.ConfigurationError, socket.error) as e:  # pragma: no cover
         # Avoid ugly, unhelpful traceback
         raise SystemExit('\n' + str(e))
 
 
 @cli.command(name="build")
-@click.option('-c', '--clean/--dirty', is_flag=True, help=clean_help)
+@click.option('-c', '--clean/--dirty', is_flag=True, default=True, help=clean_help)
 @click.option('-f', '--config-file', type=click.File('rb'), help=config_help)
 @click.option('-s', '--strict', is_flag=True, help=strict_help)
 @click.option('-t', '--theme', type=click.Choice(theme_choices), help=theme_help)
@@ -153,13 +153,13 @@ def build_command(clean, config_file, strict, theme, theme_dir, site_dir):
             theme_dir=theme_dir,
             site_dir=site_dir
         ), dirty=not clean)
-    except exceptions.ConfigurationError as e:
+    except exceptions.ConfigurationError as e:  # pragma: no cover
         # Avoid ugly, unhelpful traceback
         raise SystemExit('\n' + str(e))
 
 
 @cli.command(name="json")
-@click.option('-c', '--clean', is_flag=True, help=clean_help)
+@click.option('-c', '--clean/--dirty', is_flag=True, default=True, help=clean_help)
 @click.option('-f', '--config-file', type=click.File('rb'), help=config_help)
 @click.option('-s', '--strict', is_flag=True, help=strict_help)
 @click.option('-d', '--site-dir', type=click.Path(), help=site_dir_help)
@@ -187,13 +187,13 @@ def json_command(clean, config_file, strict, site_dir):
             strict=strict,
             site_dir=site_dir
         ), dump_json=True, dirty=not clean)
-    except exceptions.ConfigurationError as e:
+    except exceptions.ConfigurationError as e:  # pragma: no cover
         # Avoid ugly, unhelpful traceback
         raise SystemExit('\n' + str(e))
 
 
 @cli.command(name="gh-deploy")
-@click.option('-c', '--clean', is_flag=True, help=clean_help)
+@click.option('-c', '--clean/--dirty', is_flag=True, default=True, help=clean_help)
 @click.option('-f', '--config-file', type=click.File('rb'), help=config_help)
 @click.option('-m', '--message', help=commit_message_help)
 @click.option('-b', '--remote-branch', help=remote_branch_help)
@@ -209,7 +209,7 @@ def gh_deploy_command(config_file, clean, message, remote_branch, remote_name):
         )
         build.build(cfg, dirty=not clean)
         gh_deploy.gh_deploy(cfg, message=message)
-    except exceptions.ConfigurationError as e:
+    except exceptions.ConfigurationError as e:  # pragma: no cover
         # Avoid ugly, unhelpful traceback
         raise SystemExit('\n' + str(e))
 
@@ -221,5 +221,5 @@ def new_command(project_directory):
     """Create a new MkDocs project"""
     new.new(project_directory)
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     cli()


Test patch:
diff --git a/mkdocs/tests/cli_tests.py b/mkdocs/tests/cli_tests.py
index a4da76e8..0014dc2a 100644
--- a/mkdocs/tests/cli_tests.py
+++ b/mkdocs/tests/cli_tests.py
@@ -4,11 +4,16 @@
 from __future__ import unicode_literals
 import unittest
 import mock
+import logging
+import sys
+import io
 
 from click.testing import CliRunner
 
 from mkdocs import __main__ as cli
 
+PY3 = sys.version_info[0] == 3
+
 
 class CLITests(unittest.TestCase):
 
@@ -16,10 +21,10 @@ class CLITests(unittest.TestCase):
         self.runner = CliRunner()
 
     @mock.patch('mkdocs.commands.serve.serve', autospec=True)
-    def test_serve(self, mock_serve):
+    def test_serve_default(self, mock_serve):
 
         result = self.runner.invoke(
-            cli.cli, ["serve", ], catch_exceptions=False)
+            cli.cli, ["serve"], catch_exceptions=False)
 
         self.assertEqual(result.exit_code, 0)
         mock_serve.assert_called_once_with(
@@ -28,18 +33,149 @@ class CLITests(unittest.TestCase):
             strict=None,
             theme=None,
             theme_dir=None,
-            livereload=None
+            livereload='livereload'
+        )
+
+    @mock.patch('mkdocs.commands.serve.serve', autospec=True)
+    def test_serve_config_file(self, mock_serve):
+
+        result = self.runner.invoke(
+            cli.cli, ["serve", "--config-file", "mkdocs.yml"], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_serve.call_count, 1)
+        args, kwargs = mock_serve.call_args
+        self.assertTrue('config_file' in kwargs)
+        if PY3:
+            self.assertIsInstance(kwargs['config_file'], io.BufferedReader)
+        else:
+            self.assertTrue(isinstance(kwargs['config_file'], file))
+        self.assertEqual(kwargs['config_file'].name, 'mkdocs.yml')
+
+    @mock.patch('mkdocs.commands.serve.serve', autospec=True)
+    def test_serve_dev_addr(self, mock_serve):
+
+        result = self.runner.invoke(
+            cli.cli, ["serve", '--dev-addr', '0.0.0.0:80'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        mock_serve.assert_called_once_with(
+            config_file=None,
+            dev_addr='0.0.0.0:80',
+            strict=None,
+            theme=None,
+            theme_dir=None,
+            livereload='livereload'
+        )
+
+    @mock.patch('mkdocs.commands.serve.serve', autospec=True)
+    def test_serve_strict(self, mock_serve):
+
+        result = self.runner.invoke(
+            cli.cli, ["serve", '--strict'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        mock_serve.assert_called_once_with(
+            config_file=None,
+            dev_addr=None,
+            strict=True,
+            theme=None,
+            theme_dir=None,
+            livereload='livereload'
+        )
+
+    @mock.patch('mkdocs.commands.serve.serve', autospec=True)
+    def test_serve_theme(self, mock_serve):
+
+        result = self.runner.invoke(
+            cli.cli, ["serve", '--theme', 'readthedocs'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        mock_serve.assert_called_once_with(
+            config_file=None,
+            dev_addr=None,
+            strict=None,
+            theme='readthedocs',
+            theme_dir=None,
+            livereload='livereload'
+        )
+
+    @mock.patch('mkdocs.commands.serve.serve', autospec=True)
+    def test_serve_theme_dir(self, mock_serve):
+
+        result = self.runner.invoke(
+            cli.cli, ["serve", '--theme-dir', 'custom'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        mock_serve.assert_called_once_with(
+            config_file=None,
+            dev_addr=None,
+            strict=None,
+            theme=None,
+            theme_dir='custom',
+            livereload='livereload'
+        )
+
+    @mock.patch('mkdocs.commands.serve.serve', autospec=True)
+    def test_serve_livereload(self, mock_serve):
+
+        result = self.runner.invoke(
+            cli.cli, ["serve", '--livereload'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        mock_serve.assert_called_once_with(
+            config_file=None,
+            dev_addr=None,
+            strict=None,
+            theme=None,
+            theme_dir=None,
+            livereload='livereload'
+        )
+
+    @mock.patch('mkdocs.commands.serve.serve', autospec=True)
+    def test_serve_no_livereload(self, mock_serve):
+
+        result = self.runner.invoke(
+            cli.cli, ["serve", '--no-livereload'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        mock_serve.assert_called_once_with(
+            config_file=None,
+            dev_addr=None,
+            strict=None,
+            theme=None,
+            theme_dir=None,
+            livereload='no-livereload'
+        )
+
+    @mock.patch('mkdocs.commands.serve.serve', autospec=True)
+    def test_serve_dirtyreload(self, mock_serve):
+
+        result = self.runner.invoke(
+            cli.cli, ["serve", '--dirtyreload'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        mock_serve.assert_called_once_with(
+            config_file=None,
+            dev_addr=None,
+            strict=None,
+            theme=None,
+            theme_dir=None,
+            livereload='dirty'
         )
 
     @mock.patch('mkdocs.config.load_config', autospec=True)
     @mock.patch('mkdocs.commands.build.build', autospec=True)
-    def test_build(self, mock_build, mock_load_config):
+    def test_build_defaults(self, mock_build, mock_load_config):
 
         result = self.runner.invoke(
-            cli.cli, ["build", ], catch_exceptions=False)
+            cli.cli, ['build'], catch_exceptions=False)
 
         self.assertEqual(result.exit_code, 0)
         self.assertEqual(mock_build.call_count, 1)
+        args, kwargs = mock_build.call_args
+        self.assertTrue('dirty' in kwargs)
+        self.assertFalse(kwargs['dirty'])
         mock_load_config.assert_called_once_with(
             config_file=None,
             strict=None,
@@ -47,15 +183,140 @@ class CLITests(unittest.TestCase):
             theme_dir=None,
             site_dir=None
         )
+        logger = logging.getLogger('mkdocs')
+        self.assertEqual(logger.level, logging.INFO)
+
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    def test_build_clean(self, mock_build):
+
+        result = self.runner.invoke(
+            cli.cli, ['build', '--clean'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_build.call_count, 1)
+        args, kwargs = mock_build.call_args
+        self.assertTrue('dirty' in kwargs)
+        self.assertFalse(kwargs['dirty'])
+
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    def test_build_dirty(self, mock_build):
+
+        result = self.runner.invoke(
+            cli.cli, ['build', '--dirty'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_build.call_count, 1)
+        args, kwargs = mock_build.call_args
+        self.assertTrue('dirty' in kwargs)
+        self.assertTrue(kwargs['dirty'])
+
+    @mock.patch('mkdocs.config.load_config', autospec=True)
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    def test_build_config_file(self, mock_build, mock_load_config):
+
+        result = self.runner.invoke(
+            cli.cli, ['build', '--config-file', 'mkdocs.yml'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_build.call_count, 1)
+        self.assertEqual(mock_load_config.call_count, 1)
+        args, kwargs = mock_load_config.call_args
+        self.assertTrue('config_file' in kwargs)
+        if PY3:
+            self.assertIsInstance(kwargs['config_file'], io.BufferedReader)
+        else:
+            self.assertTrue(isinstance(kwargs['config_file'], file))
+        self.assertEqual(kwargs['config_file'].name, 'mkdocs.yml')
+
+    @mock.patch('mkdocs.config.load_config', autospec=True)
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    def test_build_strict(self, mock_build, mock_load_config):
+
+        result = self.runner.invoke(
+            cli.cli, ['build', '--strict'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_build.call_count, 1)
+        mock_load_config.assert_called_once_with(
+            config_file=None,
+            strict=True,
+            theme=None,
+            theme_dir=None,
+            site_dir=None
+        )
+
+    @mock.patch('mkdocs.config.load_config', autospec=True)
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    def test_build_theme(self, mock_build, mock_load_config):
+
+        result = self.runner.invoke(
+            cli.cli, ['build', '--theme', 'readthedocs'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_build.call_count, 1)
+        mock_load_config.assert_called_once_with(
+            config_file=None,
+            strict=None,
+            theme='readthedocs',
+            theme_dir=None,
+            site_dir=None
+        )
+
+    @mock.patch('mkdocs.config.load_config', autospec=True)
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    def test_build_theme_dir(self, mock_build, mock_load_config):
+
+        result = self.runner.invoke(
+            cli.cli, ['build', '--theme-dir', 'custom'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_build.call_count, 1)
+        mock_load_config.assert_called_once_with(
+            config_file=None,
+            strict=None,
+            theme=None,
+            theme_dir='custom',
+            site_dir=None
+        )
+
+    @mock.patch('mkdocs.config.load_config', autospec=True)
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    def test_build_site_dir(self, mock_build, mock_load_config):
+
+        result = self.runner.invoke(
+            cli.cli, ['build', '--site-dir', 'custom'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_build.call_count, 1)
+        mock_load_config.assert_called_once_with(
+            config_file=None,
+            strict=None,
+            theme=None,
+            theme_dir=None,
+            site_dir='custom'
+        )
 
     @mock.patch('mkdocs.commands.build.build', autospec=True)
     def test_build_verbose(self, mock_build):
 
         result = self.runner.invoke(
-            cli.cli, ["--verbose", "build"], catch_exceptions=False)
+            cli.cli, ['build', '--verbose'], catch_exceptions=False)
 
         self.assertEqual(result.exit_code, 0)
         self.assertEqual(mock_build.call_count, 1)
+        logger = logging.getLogger('mkdocs')
+        self.assertEqual(logger.level, logging.DEBUG)
+
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    def test_build_quiet(self, mock_build):
+
+        result = self.runner.invoke(
+            cli.cli, ['build', '--quiet'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_build.call_count, 1)
+        logger = logging.getLogger('mkdocs')
+        self.assertEqual(logger.level, logging.ERROR)
 
     @mock.patch('mkdocs.commands.build.build', autospec=True)
     def test_json(self, mock_build):
@@ -73,13 +334,125 @@ class CLITests(unittest.TestCase):
             cli.cli, ["new", "project"], catch_exceptions=False)
 
         self.assertEqual(result.exit_code, 0)
-        self.assertEqual(mock_new.call_count, 1)
+        mock_new.assert_called_once_with('project')
+
+    @mock.patch('mkdocs.config.load_config', autospec=True)
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    @mock.patch('mkdocs.commands.gh_deploy.gh_deploy', autospec=True)
+    def test_gh_deploy_defaults(self, mock_gh_deploy, mock_build, mock_load_config):
+
+        result = self.runner.invoke(
+            cli.cli, ['gh-deploy'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_gh_deploy.call_count, 1)
+        g_args, g_kwargs = mock_gh_deploy.call_args
+        self.assertTrue('message' in g_kwargs)
+        self.assertEqual(g_kwargs['message'], None)
+        self.assertEqual(mock_build.call_count, 1)
+        b_args, b_kwargs = mock_build.call_args
+        self.assertTrue('dirty' in b_kwargs)
+        self.assertFalse(b_kwargs['dirty'])
+        mock_load_config.assert_called_once_with(
+            config_file=None,
+            remote_branch=None,
+            remote_name=None
+        )
+
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    @mock.patch('mkdocs.commands.gh_deploy.gh_deploy', autospec=True)
+    def test_gh_deploy_clean(self, mock_gh_deploy, mock_build):
+
+        result = self.runner.invoke(
+            cli.cli, ['gh-deploy', '--clean'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_gh_deploy.call_count, 1)
+        self.assertEqual(mock_build.call_count, 1)
+        args, kwargs = mock_build.call_args
+        self.assertTrue('dirty' in kwargs)
+        self.assertFalse(kwargs['dirty'])
+
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    @mock.patch('mkdocs.commands.gh_deploy.gh_deploy', autospec=True)
+    def test_gh_deploy_dirty(self, mock_gh_deploy, mock_build):
+
+        result = self.runner.invoke(
+            cli.cli, ['gh-deploy', '--dirty'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_gh_deploy.call_count, 1)
+        self.assertEqual(mock_build.call_count, 1)
+        args, kwargs = mock_build.call_args
+        self.assertTrue('dirty' in kwargs)
+        self.assertTrue(kwargs['dirty'])
+
+    @mock.patch('mkdocs.config.load_config', autospec=True)
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    @mock.patch('mkdocs.commands.gh_deploy.gh_deploy', autospec=True)
+    def test_gh_deploy_config_file(self, mock_gh_deploy, mock_build, mock_load_config):
+
+        result = self.runner.invoke(
+            cli.cli, ['gh-deploy', '--config-file', 'mkdocs.yml'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_gh_deploy.call_count, 1)
+        self.assertEqual(mock_build.call_count, 1)
+        self.assertEqual(mock_load_config.call_count, 1)
+        args, kwargs = mock_load_config.call_args
+        self.assertTrue('config_file' in kwargs)
+        if PY3:
+            self.assertIsInstance(kwargs['config_file'], io.BufferedReader)
+        else:
+            self.assertTrue(isinstance(kwargs['config_file'], file))
+        self.assertEqual(kwargs['config_file'].name, 'mkdocs.yml')
+
+    @mock.patch('mkdocs.config.load_config', autospec=True)
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    @mock.patch('mkdocs.commands.gh_deploy.gh_deploy', autospec=True)
+    def test_gh_deploy_message(self, mock_gh_deploy, mock_build, mock_load_config):
+
+        result = self.runner.invoke(
+            cli.cli, ['gh-deploy', '--message', 'A commit message'], catch_exceptions=False)
 
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_gh_deploy.call_count, 1)
+        g_args, g_kwargs = mock_gh_deploy.call_args
+        self.assertTrue('message' in g_kwargs)
+        self.assertEqual(g_kwargs['message'], 'A commit message')
+        self.assertEqual(mock_build.call_count, 1)
+        self.assertEqual(mock_load_config.call_count, 1)
+
+    @mock.patch('mkdocs.config.load_config', autospec=True)
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
     @mock.patch('mkdocs.commands.gh_deploy.gh_deploy', autospec=True)
-    def test_gh_deploy(self, mock_gh_deploy):
+    def test_gh_deploy_remote_branch(self, mock_gh_deploy, mock_build, mock_load_config):
 
         result = self.runner.invoke(
-            cli.cli, ["gh-deploy"], catch_exceptions=False)
+            cli.cli, ['gh-deploy', '--remote-branch', 'foo'], catch_exceptions=False)
 
         self.assertEqual(result.exit_code, 0)
         self.assertEqual(mock_gh_deploy.call_count, 1)
+        self.assertEqual(mock_build.call_count, 1)
+        mock_load_config.assert_called_once_with(
+            config_file=None,
+            remote_branch='foo',
+            remote_name=None
+        )
+
+    @mock.patch('mkdocs.config.load_config', autospec=True)
+    @mock.patch('mkdocs.commands.build.build', autospec=True)
+    @mock.patch('mkdocs.commands.gh_deploy.gh_deploy', autospec=True)
+    def test_gh_deploy_remote_name(self, mock_gh_deploy, mock_build, mock_load_config):
+
+        result = self.runner.invoke(
+            cli.cli, ['gh-deploy', '--remote-name', 'foo'], catch_exceptions=False)
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(mock_gh_deploy.call_count, 1)
+        self.assertEqual(mock_build.call_count, 1)
+        mock_load_config.assert_called_once_with(
+            config_file=None,
+            remote_branch=None,
+            remote_name='foo'
+        )