Repository: PrefectHQ/prefect

Issue:
Interpolated config options are computed prior to loading user configs
In `config.toml`:

```
[section]
host = "x"
endpoint = "${section.host}:1000"
```

In `~/.prefect/config.toml`:
```
[section]
host = "y"
```

In Prefect:
```
print(prefect.config.section.endpoint) # "x:1000"
```

We should perform variable interpolation AFTER user configs are loaded


Discussion:

Resolving patch:
diff --git a/src/prefect/configuration.py b/src/prefect/configuration.py
--- a/src/prefect/configuration.py
+++ b/src/prefect/configuration.py
@@ -165,7 +165,7 @@ def string_to_type(val: str) -> Union[bool, int, float, str]:
     return val
 
 
-def interpolate_env_var(env_var: str) -> Optional[Union[bool, int, float, str]]:
+def interpolate_env_vars(env_var: str) -> Optional[Union[bool, int, float, str]]:
     """
     Expands (potentially nested) env vars by repeatedly applying
     `expandvars` and `expanduser` until interpolation stops having
@@ -196,7 +196,7 @@ def create_user_config(dest_path: str, source_path: str = DEFAULT_CONFIG) -> Non
     """
     Copies the default configuration to a user-customizable file at `dest_path`
     """
-    dest_path = cast(str, interpolate_env_var(dest_path))
+    dest_path = cast(str, interpolate_env_vars(dest_path))
     if os.path.isfile(dest_path):
         raise ValueError("File already exists: {}".format(dest_path))
     os.makedirs(os.path.dirname(dest_path), exist_ok=True)
@@ -252,18 +252,21 @@ def validate_config(config: Config) -> None:
 # Load configuration ----------------------------------------------------------
 
 
-def load_config_file(path: str, env_var_prefix: str = None, env: dict = None) -> Config:
+def load_toml(path: str) -> dict:
     """
-    Loads a configuration file from a path, optionally merging it into an existing
-    configuration.
+    Loads a config dictionary from TOML
     """
-
-    # load the configuration file
-    config = {
+    return {
         key.lower(): value
-        for key, value in toml.load(cast(str, interpolate_env_var(path))).items()
+        for key, value in toml.load(cast(str, interpolate_env_vars(path))).items()
     }
 
+
+def interpolate_config(config: dict, env_var_prefix: str = None) -> Config:
+    """
+    Processes a config dictionary, such as the one loaded from `load_toml`.
+    """
+
     # toml supports nested dicts, so we work with a flattened representation to do any
     # requested interpolation
     flat_config = collections.dict_to_flatdict(config)
@@ -273,10 +276,9 @@ def load_config_file(path: str, env_var_prefix: str = None, env: dict = None) ->
     #     [ENV_VAR_PREFIX]__[Section]__[Optional Sub-Sections...]__[Key] = Value
     # and if it does, add it to the config file.
 
-    env = cast(dict, env or os.environ)
     if env_var_prefix:
 
-        for env_var in env:
+        for env_var, env_var_value in os.environ.items():
             if env_var.startswith(env_var_prefix + "__"):
 
                 # strip the prefix off the env var
@@ -289,19 +291,19 @@ def load_config_file(path: str, env_var_prefix: str = None, env: dict = None) ->
                 # env vars with escaped characters are interpreted as literal "\", which
                 # Python helpfully escapes with a second "\". This step makes sure that
                 # escaped characters are properly interpreted.
-                value = cast(str, env.get(env_var)).encode().decode("unicode_escape")
+                value = cast(str, env_var_value.encode().decode("unicode_escape"))
 
                 # place the env var in the flat config as a compound key
                 config_option = collections.CompoundKey(
                     env_var_option.lower().split("__")
                 )
                 flat_config[config_option] = string_to_type(
-                    cast(str, interpolate_env_var(value))
+                    cast(str, interpolate_env_vars(value))
                 )
 
     # interpolate any env vars referenced
     for k, v in list(flat_config.items()):
-        flat_config[k] = interpolate_env_var(v)
+        flat_config[k] = interpolate_env_vars(v)
 
     # --------------------- Interpolate other config keys -----------------
     # TOML doesn't support references to other keys... but we do!
@@ -349,43 +351,42 @@ def load_config_file(path: str, env_var_prefix: str = None, env: dict = None) ->
 
 
 def load_configuration(
-    config_path: str,
-    env_var_prefix: str = None,
-    merge_into_config: Config = None,
-    env: dict = None,
+    path: str, user_config_path: str = None, env_var_prefix: str = None
 ) -> Config:
     """
-    Given a `config_path` with a toml configuration file, returns a Config object.
+    Loads a configuration from a known location.
 
     Args:
-        - config_path (str): the path to the toml configuration file
-        - env_var_prefix (str): if provided, environment variables starting with this prefix
-            will be added as configuration settings.
-        - merge_into_config (Config): if provided, the configuration loaded from
-            `config_path` will be merged into a copy of this configuration file. The merged
-            Config is returned.
+        - path (str): the path to the TOML configuration file
+        - user_config_path (str): an optional path to a user config file. If not provided,
+            the main config will be checked for a `user_config_path` key. If a user config
+            is provided, it will be used to update the main config prior to interpolation
+        - env_var_prefix (str): any env vars matching this prefix will be used to create
+            configuration values
+
+    Returns:
+        - Config
     """
 
     # load default config
-    config = load_config_file(config_path, env_var_prefix=env_var_prefix or "", env=env)
-
-    if merge_into_config is not None:
-        config = cast(Config, collections.merge_dicts(merge_into_config, config))
-
+    default_config = load_toml(path)
+
+    # load user config
+    if not user_config_path:
+        user_config_path = default_config.get("user_config_path", None)
+
+    if user_config_path and os.path.isfile(str(interpolate_env_vars(user_config_path))):
+        user_config = load_toml(user_config_path)
+        # merge user config into default config
+        default_config = cast(
+            dict, collections.merge_dicts(default_config, user_config)
+        )
+
+    # interpolate after user config has already been merged
+    config = interpolate_config(default_config, env_var_prefix=env_var_prefix)
+    config = process_task_defaults(config)
     validate_config(config)
-
     return config
 
 
-config = load_configuration(config_path=DEFAULT_CONFIG, env_var_prefix=ENV_VAR_PREFIX)
-
-# if user config exists, load and merge it with default config
-user_config_path = config.get("user_config_path", None)
-if user_config_path and os.path.isfile(user_config_path):
-    config = load_configuration(
-        config_path=user_config_path,
-        env_var_prefix=ENV_VAR_PREFIX,
-        merge_into_config=config,
-    )
-
-config = process_task_defaults(config)
+config = load_configuration(path=DEFAULT_CONFIG, env_var_prefix=ENV_VAR_PREFIX)


Test patch:
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -114,19 +114,23 @@ def test_config_file_path():
 
 
 @pytest.fixture
-def config(test_config_file_path):
-    environ = {}
-    environ["PREFECT__ENV_VARS__NEW_KEY"] = "TEST"
-    environ["PREFECT__ENV_VARS__TWICE__NESTED__NEW_KEY"] = "TEST"
-    environ["PREFECT__ENV_VARS__TRUE"] = "true"
-    environ["PREFECT__ENV_VARS__FALSE"] = "false"
-    environ["PREFECT__ENV_VARS__INT"] = "10"
-    environ["PREFECT__ENV_VARS__NEGATIVE_INT"] = "-10"
-    environ["PREFECT__ENV_VARS__FLOAT"] = "7.5"
-    environ["PREFECT__ENV_VARS__NEGATIVE_FLOAT"] = "-7.5"
-    environ["PREFECT__ENV_VARS__ESCAPED_CHARACTERS"] = r"line 1\nline 2\rand 3\tand 4"
-    yield configuration.load_config_file(
-        path=test_config_file_path, env_var_prefix="PREFECT", env=environ
+def config(test_config_file_path, monkeypatch):
+
+    monkeypatch.setenv("PREFECT_TEST__ENV_VARS__NEW_KEY", "TEST")
+    monkeypatch.setenv("PREFECT_TEST__ENV_VARS__TWICE__NESTED__NEW_KEY", "TEST")
+    monkeypatch.setenv("PREFECT_TEST__ENV_VARS__TRUE", "true")
+    monkeypatch.setenv("PREFECT_TEST__ENV_VARS__FALSE", "false")
+    monkeypatch.setenv("PREFECT_TEST__ENV_VARS__INT", "10")
+    monkeypatch.setenv("PREFECT_TEST__ENV_VARS__NEGATIVE_INT", "-10")
+    monkeypatch.setenv("PREFECT_TEST__ENV_VARS__FLOAT", "7.5")
+    monkeypatch.setenv("PREFECT_TEST__ENV_VARS__NEGATIVE_FLOAT", "-7.5")
+    monkeypatch.setenv("PATH", "1/2/3")
+    monkeypatch.setenv(
+        "PREFECT_TEST__ENV_VARS__ESCAPED_CHARACTERS", r"line 1\nline 2\rand 3\tand 4"
+    )
+
+    yield configuration.load_configuration(
+        test_config_file_path, env_var_prefix="PREFECT_TEST"
     )
 
 
@@ -140,7 +144,7 @@ def test_keys(config):
 def test_repr(config):
     assert (
         repr(config)
-        == "<Config: 'all_caps', 'debug', 'env_vars', 'general', 'interpolation', 'logging', 'secrets'>"
+        == "<Config: 'all_caps', 'debug', 'env_vars', 'general', 'interpolation', 'logging', 'secrets', 'tasks'>"
     )
     assert repr(config.general) == "<Config: 'nested', 'x', 'y'>"
 
@@ -176,7 +180,7 @@ def test_interpolation(config):
 
 
 def test_env_var_interpolation(config):
-    assert config.env_vars.interpolated_path == os.getenv("PATH")
+    assert config.env_vars.interpolated_path == os.environ.get("PATH")
 
 
 def test_string_to_type_function():
@@ -233,28 +237,6 @@ def test_env_var_escaped(config):
     assert config.env_vars.escaped_characters == "line 1\nline 2\rand 3\tand 4"
 
 
-def test_env_var_newline_declared_inline(config):
-    result = subprocess.check_output(
-        r'PREFECT__ENV_VARS__X="line 1\nline 2\rand 3\tand 4" python -c "import prefect; print(prefect.config.env_vars.x)"',
-        shell=True,
-    )
-    assert result.strip() == b"line 1\nline 2\rand 3\tand 4"
-
-
-def test_merge_configurations(test_config_file_path):
-
-    default_config = configuration.config
-
-    assert default_config.logging.format != "log-format"
-
-    config = configuration.load_configuration(
-        config_path=test_config_file_path, merge_into_config=default_config
-    )
-
-    assert config.logging.format == "log-format"
-    assert config.interpolation.value == 1
-
-
 def test_copy_leaves_values_mutable(config):
 
     config.set_nested("x.y.z", [1])
@@ -286,13 +268,20 @@ def test_load_user_config(self, test_config_file_path):
                 """
             )
             user_config.seek(0)
-            test_config = configuration.load_configuration(test_config_file_path)
             config = configuration.load_configuration(
-                user_config.name, merge_into_config=test_config
+                test_config_file_path, user_config.name
             )
+
+            # check that user values are loaded
             assert config.general.x == 2
             assert config.user.foo == "bar"
 
+            # check that default values are preserved
+            assert config.general.y == "hi"
+
+            # check that interpolation takes place after user config is loaded
+            assert config.general.nested.x == 2
+
 
 class TestProcessTaskDefaults:
     def test_max_retries_is_0_if_not_set(self):