Repository: EFForg/starttls-policy-cli

Issue:
Implement Early Adopter mode
Actual issue is created for tracking implementation progress of feature described [here](https://github.com/EFForg/starttls-everywhere/issues/124).

Comments:

Patch:
diff --git a/starttls_policy_cli/configure.py b/starttls_policy_cli/configure.py
index 8ec010d..bb13dec 100644
--- a/starttls_policy_cli/configure.py
+++ b/starttls_policy_cli/configure.py
@@ -20,8 +20,9 @@ class ConfigGenerator(object):
     """
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self, policy_dir):
+    def __init__(self, policy_dir, enforce_testing=False):
         self._policy_dir = policy_dir
+        self._enforce_testing = enforce_testing
         self._policy_filename = os.path.join(self._policy_dir, constants.POLICY_FILENAME)
         self._config_filename = os.path.join(self._policy_dir, self.default_filename)
         self._policy_config = None
@@ -86,15 +87,6 @@ class ConfigGenerator(object):
     def default_filename(self):
         """The expected default filename of the generated configuration file."""
 
-def _policy_for_domain(domain, tls_policy, max_domain_len):
-    line = ("{0:%d} " % max_domain_len).format(domain)
-    if tls_policy.mode == "enforce":
-        line += " secure match="
-        line += ":".join(tls_policy.mxs)
-    elif tls_policy.mode == "testing":
-        line = "# " + line + "undefined due to testing policy"
-    return line
-
 class PostfixGenerator(ConfigGenerator):
     """Configuration generator for postfix.
     """
@@ -103,7 +95,7 @@ class PostfixGenerator(ConfigGenerator):
         policies = []
         max_domain_len = len(max(policy_list, key=len))
         for domain, tls_policy in sorted(six.iteritems(policy_list)):
-            policies.append(_policy_for_domain(domain, tls_policy, max_domain_len))
+            policies.append(self._policy_for_domain(domain, tls_policy, max_domain_len))
         return "\n".join(policies)
 
     def _generate_expired_fallback(self, policy_list):
@@ -122,6 +114,16 @@ class PostfixGenerator(ConfigGenerator):
             "And finally:\n\n"
             "postfix reload\n").format(abs_path=abs_path, filename=filename)
 
+    def _policy_for_domain(self, domain, tls_policy, max_domain_len):
+        line = ("{0:%d} " % max_domain_len).format(domain)
+        mode = tls_policy.mode
+        if mode == "enforce" or self._enforce_testing and mode == "testing":
+            line += " secure match="
+            line += ":".join(tls_policy.mxs)
+        elif mode == "testing":
+            line = "# " + line + "undefined due to testing policy"
+        return line
+
     @property
     def mta_name(self):
         return "Postfix"
diff --git a/starttls_policy_cli/main.py b/starttls_policy_cli/main.py
index 2963964..fe553e6 100644
--- a/starttls_policy_cli/main.py
+++ b/starttls_policy_cli/main.py
@@ -9,12 +9,26 @@ GENERATORS = {
 }
 
 def _argument_parser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--generate", help="The MTA you want to generate a configuration file for.",
+    parser = argparse.ArgumentParser(
+        description="Generates MTA configuration file according to STARTTLS-Everywhere policy",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("-g", "--generate",
+                        choices=GENERATORS,
+                        help="The MTA you want to generate a configuration file for.",
                         dest="generate", required=True)
     # TODO: decide whether to use /etc/ for policy list home
-    parser.add_argument("--policy-dir", help="Policy file directory on this computer.",
+    parser.add_argument("-d", "--policy-dir",
+                        help="Policy file directory on this computer.",
                         default="/etc/starttls-policy/", dest="policy_dir")
+    parser.add_argument("-e", "--early-adopter",
+                        help="Early Adopter mode. Processes all \"testing\" domains in policy list "
+                        "same way as domains in \"enforce\" mode, effectively requiring strong TLS "
+                        "for domains in \"testing\" mode too. This mode is useful for participating"
+                        " in tests of recently added domains with real communications and earlier "
+                        "security hardening at the cost of increased probability of delivery "
+                        "degradation. Use this mode with awareness about all implications.",
+                        action="store_true",
+                        dest="early_adopter")
     return parser
 
 
@@ -24,19 +38,15 @@ def _ensure_directory(directory):
 
 def _generate(arguments):
     _ensure_directory(arguments.policy_dir)
-    config_generator = GENERATORS[arguments.generate](arguments.policy_dir)
+    config_generator = GENERATORS[arguments.generate](arguments.policy_dir,
+                                                      arguments.early_adopter)
     config_generator.generate()
     config_generator.manual_instructions()
 
-def _perform(arguments, parser):
-    if arguments.generate not in GENERATORS:
-        parser.error("no configuration generator exists for '%s'" % arguments.generate)
-    _generate(arguments)
-
 def main():
     """ Entrypoint for CLI tool. """
     parser = _argument_parser()
-    _perform(parser.parse_args(), parser)
+    _generate(parser.parse_args())
 
 if __name__ == "__main__":
     main()  # pragma: no cover
diff --git a/starttls_policy_cli/policy.py b/starttls_policy_cli/policy.py
index b4ab17a..1cb0212 100644
--- a/starttls_policy_cli/policy.py
+++ b/starttls_policy_cli/policy.py
@@ -1,5 +1,4 @@
 """ Policy config wrapper """
-import collections
 import logging
 import datetime
 import io
@@ -8,6 +7,12 @@ import six
 from starttls_policy_cli import util
 from starttls_policy_cli import constants
 
+try:
+    # Python 3.3+
+    from collections.abc import Mapping
+except ImportError:
+    from collections import Mapping
+
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.StreamHandler())
 
@@ -210,7 +215,6 @@ class PolicyNoAlias(Policy):
     @property
     def policy_alias(self):
         """ This type of policy can't be aliased. Returns None."""
-        pass
 
     @policy_alias.setter
     def policy_alias(self, value):
@@ -218,7 +222,7 @@ class PolicyNoAlias(Policy):
         # pylint: disable=unused-argument
         raise util.ConfigError('PolicyNoAlias object cannot have policy-alias field!')
 
-class Config(MergableConfig, collections.Mapping):
+class Config(MergableConfig, Mapping):
     """Class for retrieving properties in TLS Policy config.
     If `policy_aliases` is specified, they must be set before `policies`,
     so policy format validation can work properly.
@@ -238,6 +242,7 @@ class Config(MergableConfig, collections.Mapping):
             yield domain
 
     def keys(self):
+        """ Returns iterable over policies even if this attribute is not set """
         if self.policies is None:
             return set([])
         return self.policies.keys()
diff --git a/starttls_policy_cli/util.py b/starttls_policy_cli/util.py
index 8f7da53..6b703a7 100644
--- a/starttls_policy_cli/util.py
+++ b/starttls_policy_cli/util.py
@@ -103,20 +103,14 @@ TLS_VERSIONS = ('TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3')
 ENFORCE_MODES = ('testing', 'enforce')
 
 POLICY_SCHEMA = {
-        'min-tls-version': {
-            'enforce': partial(enforce_in, TLS_VERSIONS),
-            'default': 'TLSv1.2',
-            },
         'mode': {
             'enforce': partial(enforce_in, ENFORCE_MODES),
             'default': 'testing',
             },
-        # TODO (#50) Validate mxs as FQDNs (using public suffix list)
         'mxs': {
             'enforce': partial(enforce_list, partial(enforce_type, six.string_types)),
             'default': [],
             },
-        # TODO (#50) Validate reporting endpoint as https: or mailto:
         'policy-alias': partial(enforce_type, six.string_types),
 }
 


Test patch:
diff --git a/starttls_policy_cli/tests/configure_test.py b/starttls_policy_cli/tests/configure_test.py
index 5bd4b2e..706c16a 100644
--- a/starttls_policy_cli/tests/configure_test.py
+++ b/starttls_policy_cli/tests/configure_test.py
@@ -104,11 +104,15 @@ test_json_expired = '{\
     }'
 
 testgen_data = [
-    param("simple_policy", test_json, "# .testing.example-recipient.com "
+    param("simple_policy", test_json, False, "# .testing.example-recipient.com "
                                       "undefined due to testing policy\n"
                                       ".valid.example-recipient.com    "
                                       "secure match=.valid.example-recipient.com\n"),
-    param("expired_policy", test_json_expired, "# Policy list is outdated. "
+    param("simple_policy_early", test_json, True, ".testing.example-recipient.com  "
+                                      "secure match=.testing.example-recipient.com\n"
+                                      ".valid.example-recipient.com    "
+                                      "secure match=.valid.example-recipient.com\n"),
+    param("expired_policy", test_json_expired, False, "# Policy list is outdated. "
                                                "Falling back to opportunistic encryption.\n"),
 ]
 
@@ -127,10 +131,10 @@ class TestPostfixGenerator(unittest.TestCase):
         self.assertTrue("postfix reload" in instructions)
         self.assertTrue(generator.default_filename in instructions)
 
-    def config_test(self, conf, expected):
+    def config_test(self, conf, enforce_testing, expected):
         """PostfixGenerator test parameterized over various policies"""
         with TempPolicyDir(conf) as testdir:
-            generator = configure.PostfixGenerator(testdir)
+            generator = configure.PostfixGenerator(testdir, enforce_testing)
             generator.generate()
             pol_filename = os.path.join(testdir, generator.default_filename)
             with open(pol_filename) as pol_file:
diff --git a/starttls_policy_cli/tests/main_test.py b/starttls_policy_cli/tests/main_test.py
index 7c4a593..edca7de 100644
--- a/starttls_policy_cli/tests/main_test.py
+++ b/starttls_policy_cli/tests/main_test.py
@@ -24,21 +24,21 @@ class TestArguments(unittest.TestCase):
 
     def test_generate_arg(self):
         # pylint: disable=protected-access
-        sys.argv = ["_", "--generate", "lol"]
+        sys.argv = ["_", "--generate", "postfix"]
         parser = main._argument_parser()
         arguments = parser.parse_args()
-        self.assertEqual(arguments.generate, "lol")
+        self.assertEqual(arguments.generate, "postfix")
 
     def test_default_dir(self):
         # pylint: disable=protected-access
-        sys.argv = ["_", "--generate", "lol"]
+        sys.argv = ["_", "--generate", "postfix"]
         parser = main._argument_parser()
         arguments = parser.parse_args()
         self.assertEqual(arguments.policy_dir, "/etc/starttls-policy/")
 
     def test_policy_dir(self):
         # pylint: disable=protected-access
-        sys.argv = ["_", "--generate", "lol", "--policy-dir", "lmao"]
+        sys.argv = ["_", "--generate", "postfix", "--policy-dir", "lmao"]
         parser = main._argument_parser()
         arguments = parser.parse_args()
         self.assertEqual(arguments.policy_dir, "lmao")
@@ -50,7 +50,7 @@ class TestPerform(unittest.TestCase):
         sys.argv = ["_", "--generate", "lmao"]
         parser = main._argument_parser()
         parser.error = mock.MagicMock(side_effect=Exception)
-        self.assertRaises(Exception, main._perform, parser.parse_args(), parser)
+        self.assertRaises(Exception, parser.parse_args)
 
     @mock.patch("starttls_policy_cli.main._ensure_directory")
     def test_generate(self, ensure_directory):
@@ -58,7 +58,7 @@ class TestPerform(unittest.TestCase):
         main.GENERATORS = {"exists": mock.MagicMock()}
         sys.argv = ["_", "--generate", "exists"]
         parser = main._argument_parser()
-        main._perform(parser.parse_args(), parser)
+        main._generate(parser.parse_args())
         self.assertTrue(main.GENERATORS["exists"].called_with("/etc/starttls-policy"))
 
     @mock.patch("os.path.exists")
diff --git a/starttls_policy_cli/tests/policy_test.py b/starttls_policy_cli/tests/policy_test.py
index 81a8fb1..994c7bd 100644
--- a/starttls_policy_cli/tests/policy_test.py
+++ b/starttls_policy_cli/tests/policy_test.py
@@ -16,7 +16,6 @@ test_json = '{\
         "timestamp": "2014-05-26T01:35:33+0000",\
         "policies": {\
             ".valid.example-recipient.com": {\
-                "min-tls-version": "TLSv1.1",\
                     "mode": "enforce",\
                     "mxs": [".valid.example-recipient.com"]\
             }\
@@ -166,8 +165,8 @@ class TestConfig(unittest.TestCase):
         with self.assertRaises(util.ConfigError):
             policy.Config().policies = {'invalid': invalid_policy}
         conf = policy.Config()
-        conf.policies = {'valid': {}}
-        self.assertEqual(conf.get_policy_for('valid').min_tls_version, 'TLSv1.2')
+        conf.policies = {'valid': {'mxs': ['example.com']}}
+        self.assertEqual(conf.get_policy_for('valid').mxs, ['example.com'])
 
     def test_set_aliased_policy(self):
         conf = policy.Config()
@@ -175,7 +174,7 @@ class TestConfig(unittest.TestCase):
         with self.assertRaises(util.ConfigError):
             conf.policies = {'invalid': {'policy-alias': 'invalid'}}
         conf.policies = {'valid': {'policy-alias': 'valid'}}
-        self.assertEqual(conf.get_policy_for('valid').min_tls_version, 'TLSv1.2')
+        self.assertEqual(conf.get_policy_for('valid').mode, 'testing')
 
     def test_iter_policies_aliased(self):
         conf = policy.Config()
@@ -226,23 +225,10 @@ class TestPolicy(unittest.TestCase):
         self.assertFalse('eff.org' in new_conf.mxs)
         self.assertTrue('example.com' in new_conf.mxs)
 
-    def test_tls_version_default(self):
-        p = policy.Policy({})
-        self.assertEqual(p.min_tls_version, 'TLSv1.2')
-
     def test_mode_default(self):
         p = policy.Policy({})
         self.assertEqual(p.mode, 'testing')
 
-    def test_tls_version_valid(self):
-        with self.assertRaises(util.ConfigError):
-            policy.Policy({'min-tls-version': 'SSLv3'})
-        p = policy.Policy({})
-        with self.assertRaises(util.ConfigError):
-            p.min_tls_version = 'SSLv3'
-        p.min_tls_version = 'TLSv1.1'
-        self.assertEqual(p.min_tls_version, 'TLSv1.1')
-
     def test_mode_valid(self):
         p = policy.Policy({})
         with self.assertRaises(util.ConfigError):