Repository: dask/distributed

Issue:
Add key/value state manager to the scheduler
Cluster manager objects contain state. As experienced in `dask-gateway` and also now in `dask-ctl` if you want to destroy a cluster manager without destroying the cluster and then later create a new cluster manager to represent that existing resource the state needs to persist somewhere.

`dask-gateway` has opted to run a database alongside the service and serialize state into that database, allowing newly created cluster managers to deserialize that state.

In `dask-ctl` I have started down the road of implementing cluster manager specific serializations. For example in dask/dask-kubernetes#318 I have attempted to serialize the object into a Kubernetes Secret.

While working on that PR I found it to be very inconvenient managing the state this way, and it also came with some drawbacks. Specifically that I only serialized state at create time, so any internal state that changes over time such as the new worker name (an internal counter) is not stored.

I also have concerns that not all platforms (batch systems in particular) will have a convenient way to store cluster state.

One thing that is consistent between all clusters is the scheduler. The discovery mechanism in `dask-ctl` is intended to find schedulers (and everything else) and then reconstruct the object around them. One solution to managing state would be to sync that state over to the scheduler, so when reconstructing an object if we can find the scheduler using platform specific means and connect to it we would be able to retrieve the state from there.

Therefore I propose adding a key/value store within the scheduler that can be used via the RPC. My intended implementation would be to have a dictionary in both the `Scheduler` and `Cluster` objects which would be kept in sync via the RPC. I think it would be fine to insist that only JSON serializable objects can be stored here.


Discussion:
Having get/set routes for msgpack serailizable data in the `Scheduler`, or even down in the`Server` class sounds fine to me.  This should also be pretty trivial to implement.
The scheduler already has `get_metadata` & `set_metadata` methods / handlers. I've not looked at them in a while, but they might be of use here 

Resolving patch:
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import datetime
 import logging
 import threading
@@ -9,7 +10,7 @@
 from tornado.ioloop import PeriodicCallback
 
 import dask.config
-from dask.utils import _deprecated, format_bytes, parse_timedelta
+from dask.utils import _deprecated, format_bytes, parse_timedelta, typename
 from dask.widgets import get_template
 
 from ..core import Status
@@ -44,9 +45,8 @@ class Cluster:
     """
 
     _supports_scaling = True
-    name = None
 
-    def __init__(self, asynchronous, quiet=False, name=None):
+    def __init__(self, asynchronous, quiet=False, name=None, scheduler_sync_interval=1):
         self.scheduler_info = {"workers": {}}
         self.periodic_callbacks = {}
         self._asynchronous = asynchronous
@@ -56,13 +56,24 @@ def __init__(self, asynchronous, quiet=False, name=None):
         self.quiet = quiet
         self.scheduler_comm = None
         self._adaptive = None
+        self._sync_interval = parse_timedelta(
+            scheduler_sync_interval, default="seconds"
+        )
+
+        if name is None:
+            name = str(uuid.uuid4())[:8]
 
-        if name is not None:
-            self.name = name
-        elif self.name is None:
-            self.name = str(uuid.uuid4())[:8]
+        self._cluster_info = {"name": name, "type": typename(type(self))}
         self.status = Status.created
 
+    @property
+    def name(self):
+        return self._cluster_info["name"]
+
+    @name.setter
+    def name(self, name):
+        self._cluster_info["name"] = name
+
     async def _start(self):
         comm = await self.scheduler_comm.live_comm()
         await comm.write({"op": "subscribe_worker_status"})
@@ -71,8 +82,25 @@ async def _start(self):
         self._watch_worker_status_task = asyncio.ensure_future(
             self._watch_worker_status(comm)
         )
+
+        info = await self.scheduler_comm.get_metadata(
+            keys=["cluster-manager-info"], default={}
+        )
+        self._cluster_info.update(info)
+
+        self.periodic_callbacks["sync-cluster-info"] = PeriodicCallback(
+            self._sync_cluster_info, self._sync_interval * 1000
+        )
+        for pc in self.periodic_callbacks.values():
+            pc.start()
         self.status = Status.running
 
+    async def _sync_cluster_info(self):
+        await self.scheduler_comm.set_metadata(
+            keys=["cluster-manager-info"],
+            value=copy.copy(self._cluster_info),
+        )
+
     async def _close(self):
         if self.status == Status.closed:
             return
@@ -85,12 +113,12 @@ async def _close(self):
         if self._watch_worker_status_task:
             await self._watch_worker_status_task
 
-        for pc in self.periodic_callbacks.values():
-            pc.stop()
-
         if self.scheduler_comm:
             await self.scheduler_comm.close_rpc()
 
+        for pc in self.periodic_callbacks.values():
+            pc.stop()
+
         self.status = Status.closed
 
     def close(self, timeout=None):
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -121,6 +121,7 @@ def __init__(
         interface=None,
         worker_class=None,
         scheduler_kwargs=None,
+        scheduler_sync_interval=1,
         **worker_kwargs,
     ):
         if ip is not None:
@@ -241,6 +242,7 @@ def __init__(
             asynchronous=asynchronous,
             silence_logs=silence_logs,
             security=security,
+            scheduler_sync_interval=scheduler_sync_interval,
         )
 
     def start_worker(self, *args, **kwargs):
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -245,6 +245,7 @@ def __init__(
         silence_logs=False,
         name=None,
         shutdown_on_close=True,
+        scheduler_sync_interval=1,
     ):
         self._created = weakref.WeakSet()
 
@@ -274,6 +275,7 @@ def __init__(
         super().__init__(
             asynchronous=asynchronous,
             name=name,
+            scheduler_sync_interval=scheduler_sync_interval,
         )
 
         if not self.asynchronous:


Test patch:
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1073,3 +1073,30 @@ async def test_local_cluster_redundant_kwarg(nanny):
             async with Client(cluster) as c:
                 f = c.submit(sleep, 0)
                 await f
+
+
+@pytest.mark.asyncio
+async def test_cluster_info_sync():
+    async with LocalCluster(
+        processes=False, asynchronous=True, scheduler_sync_interval="1ms"
+    ) as cluster:
+        assert cluster._cluster_info["name"] == cluster.name
+
+        while "name" not in cluster.scheduler.get_metadata(
+            keys=["cluster-manager-info"]
+        ):
+            await asyncio.sleep(0.01)
+
+        info = await cluster.scheduler_comm.get_metadata(keys=["cluster-manager-info"])
+        assert info["name"] == cluster.name
+        info = cluster.scheduler.get_metadata(keys=["cluster-manager-info"])
+        assert info["name"] == cluster.name
+
+        cluster._cluster_info["foo"] = "bar"
+        while "foo" not in cluster.scheduler.get_metadata(
+            keys=["cluster-manager-info"]
+        ):
+            await asyncio.sleep(0.01)
+
+        info = cluster.scheduler.get_metadata(keys=["cluster-manager-info"])
+        assert info["foo"] == "bar"