Repository: dask/distributed

Issue:
Impossible to override default config with DASK_INTERNAL_INHERIT_CONFIG
**What happened**:

When passing config with `DASK_INTERNAL_INHERIT_CONFIG` it appears it is not possible to override the default values.

**What you expected to happen**:

I expected the config passed with `DASK_INTERNAL_INHERIT_CONFIG` to be applied.

**Minimal Complete Verifiable Example**:

Generate config
```python
>>> from distributed.utils import serialize_for_cli
>>> serialize_for_cli({"distributed": {"comm": {"default-scheme": "tls"}}})
'eyJkaXN0cmlidXRlZCI6IHsiY29tbSI6IHsiZGVmYXVsdC1zY2hlbWUiOiAidGxzIn19fQ=='
```

Start scheduler with `DASK_INTERNAL_INHERIT_CONFIG`.

```console
$ DASK_INTERNAL_INHERIT_CONFIG="eyJkaXN0cmlidXRlZCI6IHsiY29tbSI6IHsiZGVmYXVsdC1zY2hlbWUiOiAidGxzIn19fQ==" dask-scheduler
```

Connect a client and check the config.

```python
>>> from dask.distributed import Client
>>> import dask.config
>>> client = Client("tcp://localhost:8786")
>>> client.run_on_scheduler(lambda: dask.config.get("distributed.comm.default-scheme"))
'tcp'
```

**Anything else we need to know?**:

It appears that existing config is prioritised, even if the existing config is just detected from the defaults.

https://github.com/dask/distributed/blob/f2b09c6b55188f6b633cba5422454b689b0fd7b4/distributed/cli/dask_scheduler.py#L170-L173

**Environment**:

- Dask version: 2020.12.0+6.g1d669480 (master)
- Distributed version: 2020.12.0+7.gc2d87738 (master)
- Python version: 3.8.5
- Operating System: macOS
- Install method (conda, pip, source): Source



Discussion:
cc @madsbk 

Resolving patch:
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -169,8 +169,7 @@ def main(
 
     if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
         config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
-        # Update the global config given priority to the existing global config
-        dask.config.update(dask.config.global_config, config, priority="old")
+        dask.config.update(dask.config.global_config, config)
 
     if not host and (tls_ca_file or tls_cert or tls_key):
         host = "tls://"
diff --git a/distributed/cli/dask_spec.py b/distributed/cli/dask_spec.py
--- a/distributed/cli/dask_spec.py
+++ b/distributed/cli/dask_spec.py
@@ -1,10 +1,13 @@
 import asyncio
 import click
 import json
+import os
 import sys
 import yaml
 
+import dask.config
 from distributed.deploy.spec import run_spec
+from distributed.utils import deserialize_for_cli
 
 
 @click.command(context_settings=dict(ignore_unknown_options=True))
@@ -13,6 +16,11 @@
 @click.option("--spec-file", type=str, default=None, help="")
 @click.version_option()
 def main(args, spec: str, spec_file: str):
+
+    if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
+        config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
+        dask.config.update(dask.config.global_config, config)
+
     if spec and spec_file or not spec and not spec_file:
         print("Must specify exactly one of --spec and --spec-file")
         sys.exit(1)
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -397,8 +397,7 @@ def del_pid_file():
 
     if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
         config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
-        # Update the global config given priority to the existing global config
-        dask.config.update(dask.config.global_config, config, priority="old")
+        dask.config.update(dask.config.global_config, config)
 
     nannies = [
         t(


Test patch:
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -45,6 +45,8 @@
     LRU,
     offload,
     TimeoutError,
+    deserialize_for_cli,
+    serialize_for_cli,
 )
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
 from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
@@ -606,3 +608,19 @@ def test_lru():
 async def test_offload():
     assert (await offload(inc, 1)) == 2
     assert (await offload(lambda x, y: x + y, 1, y=2)) == 3
+
+
+def test_cli_serialization():
+    # Use context manager without changing the value to ensure test side effects are restored
+    with dask.config.set(
+        {
+            "distributed.comm.default-scheme": dask.config.get(
+                "distributed.comm.default-scheme"
+            )
+        }
+    ):
+        config = deserialize_for_cli(
+            serialize_for_cli({"distributed": {"comm": {"default-scheme": "tls"}}})
+        )  # Take a round trip through the serialization
+        dask.config.update(dask.config.global_config, config)
+        assert dask.config.get("distributed.comm.default-scheme") == "tls"