Repository: PrefectHQ/prefect

Issue:
Breaking change in 0.6.0
Hi team,

We were using release 0.5.5 and when trying to upgrade to 0.6.0 we are faced with a breaking change: prefect forces us to use all lowercase letters for keys in the config.toml file. 

This creates a problem for us because we have several keys who are different only in their letter-casing by design, so that if we try to lowercase both, we are faced with a duplicate key. 

Is there a good reason for this change or is it something that may be fixed?

Thanks!


Discussion:
Hi @orcaman - the reason for this change is that the primary way of setting configuration values is by environment variable, and depending on the OS, env vars may not respect case. Therefore it becomes impossible to update a mixed-case configuration key by environment variable. (We found this one out the hard way from some Windows users.) Moreover, we don't want to only update configuration keys that exactly match the environment variable casing because most env vars are uppercase by convention, but prefect keys are generally not.

However, we could consider relaxing the error message and documenting that environment variables can only be used to override lower-case config keys. @cicdw what do you think?



@jlowin  that would be great!

Resolving patch:
diff --git a/src/prefect/configuration.py b/src/prefect/configuration.py
--- a/src/prefect/configuration.py
+++ b/src/prefect/configuration.py
@@ -247,22 +247,11 @@ def process_task_defaults(config: Config) -> Config:
 def validate_config(config: Config) -> None:
     """
     Validates that the configuration file is valid.
-        - keys are lowercase
         - keys do not shadow Config methods
 
     Note that this is performed when the config is first loaded, but not after.
     """
 
-    def check_lowercase_keys(config: Config) -> None:
-        """
-        Recursively check that keys are lowercase
-        """
-        for k, v in config.items():
-            if k != k.lower():
-                raise ValueError('Config keys must be lowercase: "{}"'.format(k))
-            if isinstance(v, Config) and k != "context":
-                check_lowercase_keys(v)
-
     def check_valid_keys(config: Config) -> None:
         """
         Recursively check that keys do not shadow methods of the Config object
@@ -274,7 +263,6 @@ def check_valid_keys(config: Config) -> None:
             if isinstance(v, Config):
                 check_valid_keys(v)
 
-    check_lowercase_keys(config)
     check_valid_keys(config)
 
 


Test patch:
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -388,26 +388,11 @@ def test_invalid_env_var_raises_error(self, monkeypatch):
                     test_config.name, env_var_prefix="PREFECT_TEST"
                 )
 
-    def test_non_lowercase_keys_raise_error(self):
-
-        with tempfile.NamedTemporaryFile() as test_config:
-            test_config.write(
-                b"""
-                [KEY]
-                x = 1
-                """
-            )
-            test_config.seek(0)
-
-            with pytest.raises(ValueError):
-                configuration.load_configuration(test_config.name)
-
-    def test_non_lowercase_keys_are_ok_in_context(self):
-
+    def test_mixed_case_keys_are_ok(self):
         with tempfile.NamedTemporaryFile() as test_config:
             test_config.write(
                 b"""
-                [context]
+                [SeCtIoN]
                 KeY = 1
                 """
             )
@@ -415,22 +400,26 @@ def test_non_lowercase_keys_are_ok_in_context(self):
 
             config = configuration.load_configuration(test_config.name)
 
-        assert "KeY" in config.context
-        assert config.context.KeY == 1
+        assert "KeY" in config.SeCtIoN
+        assert config.SeCtIoN.KeY == 1
+
+    def test_env_vars_are_interpolated_as_lower_case(self, monkeypatch):
 
-    def test_non_lowercase_keys_are_ok_in_context_subsections(self):
+        monkeypatch.setenv("PREFECT_TEST__SECTION__KEY", "2")
 
         with tempfile.NamedTemporaryFile() as test_config:
             test_config.write(
                 b"""
-                [context.secrets]
-                API_CREDS = 4
+                [SeCtIoN]
+                KeY = 1
                 """
             )
             test_config.seek(0)
 
-            config = configuration.load_configuration(test_config.name)
+            config = configuration.load_configuration(
+                test_config.name, env_var_prefix="PREFECT_TEST"
+            )
 
-        assert "secrets" in config.context
-        assert "API_CREDS" in config.context.secrets
-        assert config.context.secrets["API_CREDS"] == 4
+        assert "KeY" in config.SeCtIoN
+        assert config.SeCtIoN.KeY == 1
+        assert config.section.key == 2