Repository: certbot/certbot

Issue:
Need to be able to define the files parsed in Apache vhost directory
While in some distributions all the configuration files should have .conf - extension, which is the default after #1811 - some distributions use filenames without extension. However for example CentOS has non-configuration files in the same directory as vhost .conf  - files, and augeas errors out on them.

We will need to add the option to define files to be parsed per-os basis.



Discussion:

Resolving patch:
diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py
--- a/letsencrypt-apache/letsencrypt_apache/configurator.py
+++ b/letsencrypt-apache/letsencrypt_apache/configurator.py
@@ -86,10 +86,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
     @classmethod
     def add_parser_arguments(cls, add):
-        add("ctl", default=constants.os_constant("ctl"),
-            help="Path to the 'apache2ctl' binary, used for 'configtest', "
-                 "retrieving the Apache2 version number, and initialization "
-                 "parameters.")
         add("enmod", default=constants.os_constant("enmod"),
             help="Path to the Apache 'a2enmod' binary.")
         add("dismod", default=constants.os_constant("dismod"),
@@ -148,10 +144,8 @@ def prepare(self):
 
         """
         # Verify Apache is installed
-        for exe in (self.conf("ctl"), self.conf("enmod"), self.conf("dismod")):
-            if exe is not None:
-                if not le_util.exe_exists(exe):
-                    raise errors.NoInstallationError
+        if not le_util.exe_exists(constants.os_constant("restart_cmd")[0]):
+            raise errors.NoInstallationError
 
         # Make sure configuration is valid
         self.config_test()
@@ -165,7 +159,7 @@ def prepare(self):
 
         self.parser = parser.ApacheParser(
             self.aug, self.conf("server-root"), self.conf("vhost-root"),
-            self.conf("ctl"), self.version)
+            self.version)
         # Check for errors in parsing files with Augeas
         self.check_parsing_errors("httpd.aug")
 
@@ -1277,7 +1271,7 @@ def enable_mod(self, mod_name, temp=False):
         # Modules can enable additional config files. Variables may be defined
         # within these new configuration sections.
         # Reload is not necessary as DUMP_RUN_CFG uses latest config.
-        self.parser.update_runtime_variables(self.conf("ctl"))
+        self.parser.update_runtime_variables()
 
     def _add_parser_mod(self, mod_name):
         """Shortcut for updating parser modules."""
@@ -1315,7 +1309,7 @@ def _reload(self):
 
         """
         try:
-            le_util.run_script([self.conf("ctl"), "graceful"])
+            le_util.run_script(constants.os_constant("restart_cmd"))
         except errors.SubprocessError as err:
             raise errors.MisconfigurationError(str(err))
 
@@ -1326,7 +1320,7 @@ def config_test(self):  # pylint: disable=no-self-use
 
         """
         try:
-            le_util.run_script([self.conf("ctl"), "configtest"])
+            le_util.run_script(constants.os_constant("conftest_cmd"))
         except errors.SubprocessError as err:
             raise errors.MisconfigurationError(str(err))
 
@@ -1346,7 +1340,8 @@ def get_version(self):
                 constants.os_constant("version_cmd"))
         except errors.SubprocessError:
             raise errors.PluginError(
-                "Unable to run %s -v" % self.conf("ctl"))
+                "Unable to run %s -v" %
+                constants.os_constant("version_cmd"))
 
         regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE)
         matches = regex.findall(stdout)
diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py
--- a/letsencrypt-apache/letsencrypt_apache/constants.py
+++ b/letsencrypt-apache/letsencrypt_apache/constants.py
@@ -6,9 +6,11 @@
 CLI_DEFAULTS_DEBIAN = dict(
     server_root="/etc/apache2",
     vhost_root="/etc/apache2/sites-available",
-    ctl="apache2ctl",
+    vhost_files="*",
     version_cmd=['apache2ctl', '-v'],
     define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'],
+    restart_cmd=['apache2ctl', 'graceful'],
+    conftest_cmd=['apache2ctl', 'configtest'],
     enmod="a2enmod",
     dismod="a2dismod",
     le_vhost_ext="-le-ssl.conf",
@@ -19,9 +21,11 @@
 CLI_DEFAULTS_CENTOS = dict(
     server_root="/etc/httpd",
     vhost_root="/etc/httpd/conf.d",
-    ctl="apachectl",
+    vhost_files="*.conf",
     version_cmd=['apachectl', '-v'],
     define_cmd=['apachectl', '-t', '-D', 'DUMP_RUN_CFG'],
+    restart_cmd=['apachectl', 'graceful'],
+    conftest_cmd=['apachectl', 'configtest'],
     enmod=None,
     dismod=None,
     le_vhost_ext="-le-ssl.conf",
@@ -32,9 +36,11 @@
 CLI_DEFAULTS_GENTOO = dict(
     server_root="/etc/apache2",
     vhost_root="/etc/apache2/vhosts.d",
-    ctl="apache2ctl",
+    vhost_files="*.conf",
     version_cmd=['/usr/sbin/apache2', '-v'],
     define_cmd=['/usr/sbin/apache2', '-t', '-D', 'DUMP_RUN_CFG'],
+    restart_cmd=['apache2ctl', 'graceful'],
+    conftest_cmd=['apache2ctl', 'configtest'],
     enmod=None,
     dismod=None,
     le_vhost_ext="-le-ssl.conf",
diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py
--- a/letsencrypt-apache/letsencrypt_apache/parser.py
+++ b/letsencrypt-apache/letsencrypt_apache/parser.py
@@ -28,7 +28,7 @@ class ApacheParser(object):
     arg_var_interpreter = re.compile(r"\$\{[^ \}]*}")
     fnmatch_chars = set(["*", "?", "\\", "[", "]"])
 
-    def __init__(self, aug, root, vhostroot, ctl, version=(2, 4)):
+    def __init__(self, aug, root, vhostroot, version=(2, 4)):
         # Note: Order is important here.
 
         # This uses the binary, so it can be done first.
@@ -37,7 +37,7 @@ def __init__(self, aug, root, vhostroot, ctl, version=(2, 4)):
         # This only handles invocation parameters and Define directives!
         self.variables = {}
         if version >= (2, 4):
-            self.update_runtime_variables(ctl)
+            self.update_runtime_variables()
 
         self.aug = aug
         # Find configuration root and make sure augeas can parse it.
@@ -60,9 +60,10 @@ def __init__(self, aug, root, vhostroot, ctl, version=(2, 4)):
         self.loc.update(self._set_locations())
 
         # Must also attempt to parse virtual host root
-        self._parse_file(self.vhostroot + "/*.conf")
+        self._parse_file(self.vhostroot + "/" +
+                         constants.os_constant("vhost_files"))
 
-        #check to see if there were unparsed define statements
+        # check to see if there were unparsed define statements
         if version < (2, 4):
             if self.find_dir("Define", exclude=False):
                 raise errors.PluginError("Error parsing runtime variables")
@@ -91,7 +92,7 @@ def init_modules(self):
                 self.modules.add(
                     os.path.basename(self.get_arg(match_filename))[:-2] + "c")
 
-    def update_runtime_variables(self, ctl):
+    def update_runtime_variables(self):
         """"
 
         .. note:: Compile time variables (apache2ctl -V) are not used within the
@@ -101,7 +102,7 @@ def update_runtime_variables(self, ctl):
         .. todo:: Create separate compile time variables... simply for arg_get()
 
         """
-        stdout = self._get_runtime_cfg(ctl)
+        stdout = self._get_runtime_cfg()
 
         variables = dict()
         matches = re.compile(r"Define: ([^ \n]*)").findall(stdout)
@@ -121,7 +122,7 @@ def update_runtime_variables(self, ctl):
 
         self.variables = variables
 
-    def _get_runtime_cfg(self, ctl):  # pylint: disable=no-self-use
+    def _get_runtime_cfg(self):  # pylint: disable=no-self-use
         """Get runtime configuration info.
 
         :returns: stdout from DUMP_RUN_CFG
@@ -136,9 +137,11 @@ def _get_runtime_cfg(self, ctl):  # pylint: disable=no-self-use
 
         except (OSError, ValueError):
             logger.error(
-                "Error accessing %s for runtime parameters!%s", ctl, os.linesep)
+                "Error running command %s for runtime parameters!%s",
+                constants.os_constant("define_cmd"), os.linesep)
             raise errors.MisconfigurationError(
-                "Error accessing loaded Apache parameters: %s", ctl)
+                "Error accessing loaded Apache parameters: %s",
+                constants.os_constant("define_cmd"))
         # Small errors that do not impede
         if proc.returncode != 0:
             logger.warn("Error in checking parameter list: %s", stderr)


Test patch:
diff --git a/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py b/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py
--- a/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py
+++ b/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py
@@ -11,14 +11,17 @@ class ConstantsTest(unittest.TestCase):
     @mock.patch("letsencrypt.le_util.get_os_info")
     def test_get_debian_value(self, os_info):
         os_info.return_value = ('Debian', '', '')
-        self.assertEqual(constants.os_constant("ctl"), "apache2ctl")
+        self.assertEqual(constants.os_constant("vhost_root"),
+                         "/etc/apache2/sites-available")
 
     @mock.patch("letsencrypt.le_util.get_os_info")
     def test_get_centos_value(self, os_info):
         os_info.return_value = ('CentOS Linux', '', '')
-        self.assertEqual(constants.os_constant("ctl"), "apachectl")
+        self.assertEqual(constants.os_constant("vhost_root"),
+                         "/etc/httpd/conf.d")
 
     @mock.patch("letsencrypt.le_util.get_os_info")
     def test_get_default_value(self, os_info):
         os_info.return_value = ('Nonexistent Linux', '', '')
-        self.assertEqual(constants.os_constant("ctl"), "apache2ctl")
+        self.assertEqual(constants.os_constant("vhost_root"),
+                         "/etc/apache2/sites-available")
diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py
--- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py
+++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py
@@ -145,24 +145,26 @@ def test_update_runtime_variables(self, mock_cfg):
         expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443",
                          "example_path": "Documents/path"}
 
-        self.parser.update_runtime_variables("ctl")
+        self.parser.update_runtime_variables()
         self.assertEqual(self.parser.variables, expected_vars)
 
     @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg")
     def test_update_runtime_vars_bad_output(self, mock_cfg):
         mock_cfg.return_value = "Define: TLS=443=24"
-        self.parser.update_runtime_variables("ctl")
+        self.parser.update_runtime_variables()
 
         mock_cfg.return_value = "Define: DUMP_RUN_CFG\nDefine: TLS=443=24"
         self.assertRaises(
-            errors.PluginError, self.parser.update_runtime_variables, "ctl")
+            errors.PluginError, self.parser.update_runtime_variables)
 
+    @mock.patch("letsencrypt_apache.constants.os_constant")
     @mock.patch("letsencrypt_apache.parser.subprocess.Popen")
-    def test_update_runtime_vars_bad_ctl(self, mock_popen):
+    def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const):
         mock_popen.side_effect = OSError
+        mock_const.return_value = "nonexistent"
         self.assertRaises(
             errors.MisconfigurationError,
-            self.parser.update_runtime_variables, "ctl")
+            self.parser.update_runtime_variables)
 
     @mock.patch("letsencrypt_apache.parser.subprocess.Popen")
     def test_update_runtime_vars_bad_exit(self, mock_popen):
@@ -170,7 +172,7 @@ def test_update_runtime_vars_bad_exit(self, mock_popen):
         mock_popen.returncode = -1
         self.assertRaises(
             errors.MisconfigurationError,
-            self.parser.update_runtime_variables, "ctl")
+            self.parser.update_runtime_variables)
 
 
 class ParserInitTest(util.ApacheTest):
@@ -191,7 +193,7 @@ def test_unparsable(self, mock_cfg):
         self.assertRaises(
             errors.PluginError,
             ApacheParser, self.aug, os.path.relpath(self.config_path),
-            "/dummy/vhostpath", "ctl", version=(2, 2, 22))
+            "/dummy/vhostpath", version=(2, 2, 22))
 
     def test_root_normalized(self):
         from letsencrypt_apache.parser import ApacheParser
@@ -203,7 +205,7 @@ def test_root_normalized(self):
                 "debian_apache_2_4/////two_vhost_80/../two_vhost_80/apache2")
 
             parser = ApacheParser(self.aug, path,
-                                  "/dummy/vhostpath", "dummy_ctl")
+                                  "/dummy/vhostpath")
 
         self.assertEqual(parser.root, self.config_path)
 
@@ -213,7 +215,7 @@ def test_root_absolute(self):
                         "update_runtime_variables"):
             parser = ApacheParser(
                 self.aug, os.path.relpath(self.config_path),
-                "/dummy/vhostpath", "dummy_ctl")
+                "/dummy/vhostpath")
 
         self.assertEqual(parser.root, self.config_path)
 
@@ -223,7 +225,7 @@ def test_root_no_trailing_slash(self):
                         "update_runtime_variables"):
             parser = ApacheParser(
                 self.aug, self.config_path + os.path.sep,
-                "/dummy/vhostpath", "dummy_ctl")
+                "/dummy/vhostpath")
         self.assertEqual(parser.root, self.config_path)
 
 
diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py
--- a/letsencrypt-apache/letsencrypt_apache/tests/util.py
+++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py
@@ -58,7 +58,7 @@ def setUp(self, test_dir="debian_apache_2_4/two_vhost_80",
         with mock.patch("letsencrypt_apache.parser.ApacheParser."
                         "update_runtime_variables"):
             self.parser = ApacheParser(
-                self.aug, self.config_path, self.vhost_path, "dummy_ctl_path")
+                self.aug, self.config_path, self.vhost_path)
 
 
 def get_apache_configurator(