Repository: zalando/patroni
Pull request #2485 (2022-12-13 10:11:03)

Introduce the failsafe key in DCS
Extracted from #2379

hughcapet: 👍 
coveralls: ## Pull Request Test Coverage Report for [Build 3684380133](https://coveralls.io/builds/55105914)

* **74** of **74**   **(100.0%)**  changed or added relevant lines in **7** files are covered.
* No unchanged relevant lines lost coverage.
* Overall coverage remained the same at **99.895%**

---



|  Totals | [![Coverage Status](https://coveralls.io/builds/55105914/badge)](https://coveralls.io/builds/55105914) |
| :-- | --: |
| Change from base [Build 3684345809](https://coveralls.io/builds/55105706): |  0% |
| Covered Lines: | 11368 |
| Relevant Lines: | 11380 |

---
##### 💛  - [Coveralls](https://coveralls.io)

CyberDem0n: 👍

Patch:
diff --git a/patroni/dcs/__init__.py b/patroni/dcs/__init__.py
index 0d414558..10242f0d 100644
--- a/patroni/dcs/__init__.py
+++ b/patroni/dcs/__init__.py
@@ -444,7 +444,7 @@ class TimelineHistory(namedtuple('TimelineHistory', 'index,value,lines')):
         return TimelineHistory(index, value, lines)
 
 
-class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,failover,sync,history,slots')):
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,failover,sync,history,slots,failsafe')):
 
     """Immutable object (namedtuple) which represents PostgreSQL cluster.
     Consists of the following fields:
@@ -606,11 +606,11 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f
     @property
     def timeline(self):
         """
-        >>> Cluster(0, 0, 0, 0, 0, 0, 0, 0, 0).timeline
+        >>> Cluster(0, 0, 0, 0, 0, 0, 0, 0, 0, None).timeline
         0
-        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]'), 0).timeline
+        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]'), 0, None).timeline
         1
-        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), 0).timeline
+        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), 0, None).timeline
         0
         """
         if self.history:
@@ -655,6 +655,7 @@ class AbstractDCS(object):
     _STATUS = 'status'  # JSON, contains "leader_lsn" and confirmed_flush_lsn of logical "slots" on the leader
     _LEADER_OPTIME = _OPTIME + '/' + _LEADER  # legacy
     _SYNC = 'sync'
+    _FAILSAFE = 'failsafe'
 
     def __init__(self, config):
         """
@@ -672,6 +673,7 @@ class AbstractDCS(object):
         self._last_lsn = ''
         self._last_seen = 0
         self._last_status = {}
+        self._last_failsafe = {}
         self.event = Event()
 
     def client_path(self, path):
@@ -717,6 +719,10 @@ class AbstractDCS(object):
     def sync_path(self):
         return self.client_path(self._SYNC)
 
+    @property
+    def failsafe_path(self):
+        return self.client_path(self._FAILSAFE)
+
     @abc.abstractmethod
     def set_ttl(self, ttl):
         """Set the new ttl value for leader key"""
@@ -768,6 +774,8 @@ class AbstractDCS(object):
             raise
 
         self._last_seen = int(time.time())
+        self._last_status = {self._OPTIME: cluster.last_lsn, 'slots': cluster.slots}
+        self._last_failsafe = cluster.failsafe
 
         with self._cluster_thread_lock:
             self._cluster = cluster
@@ -810,6 +818,18 @@ class AbstractDCS(object):
             self._last_lsn = value[self._OPTIME]
             self._write_leader_optime(str(value[self._OPTIME]))
 
+    @abc.abstractmethod
+    def _write_failsafe(self, value):
+        """Write current cluster topology to DCS that will be used by failsafe mechanism (if enabled).
+
+        :param value: failsafe topology serialized in JSON format
+        :returns: `!True` on success."""
+
+    def write_failsafe(self, value):
+        if not (isinstance(self._last_failsafe, dict) and deep_compare(self._last_failsafe, value))\
+                and self._write_failsafe(json.dumps(value, separators=(',', ':'))):
+            self._last_failsafe = value
+
     @abc.abstractmethod
     def _update_leader(self):
         """Update leader key (or session) ttl
@@ -821,7 +841,7 @@ class AbstractDCS(object):
         If update fails due to DCS not being accessible or because it is not able to
         process requests (hopefuly temporary), the ~DCSError exception should be raised."""
 
-    def update_leader(self, last_lsn, slots=None):
+    def update_leader(self, last_lsn, slots=None, failsafe=None):
         """Update leader key (or session) ttl and optime/leader
 
         :param last_lsn: absolute WAL LSN in bytes
@@ -834,6 +854,10 @@ class AbstractDCS(object):
             if slots:
                 status['slots'] = slots
             self.write_status(status)
+
+        if ret and failsafe is not None:
+            self.write_failsafe(failsafe)
+
         return ret
 
     @abc.abstractmethod
diff --git a/patroni/dcs/consul.py b/patroni/dcs/consul.py
index 1d506d4e..eb1a180b 100644
--- a/patroni/dcs/consul.py
+++ b/patroni/dcs/consul.py
@@ -389,9 +389,16 @@ class Consul(AbstractDCS):
             sync = nodes.get(self._SYNC)
             sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
 
-            return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+            # get failsafe topology
+            failsafe = nodes.get(self._FAILSAFE)
+            try:
+                failsafe = json.loads(failsafe['Value']) if failsafe else None
+            except Exception:
+                failsafe = None
+
+            return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
         except NotFound:
-            return Cluster(None, None, None, None, [], None, None, None, None)
+            return Cluster(None, None, None, None, [], None, None, None, None, None)
         except Exception:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
@@ -554,6 +561,10 @@ class Consul(AbstractDCS):
     def _write_status(self, value):
         return self._client.kv.put(self.status_path, value)
 
+    @catch_consul_errors
+    def _write_failsafe(self, value):
+        return self._client.kv.put(self.failsafe_path, value)
+
     @staticmethod
     def _run_and_handle_exceptions(method, *args, **kwargs):
         retry = kwargs.pop('retry', None)
diff --git a/patroni/dcs/etcd.py b/patroni/dcs/etcd.py
index 2921b8e1..3a8cf0c1 100644
--- a/patroni/dcs/etcd.py
+++ b/patroni/dcs/etcd.py
@@ -661,9 +661,16 @@ class Etcd(AbstractEtcd):
             sync = nodes.get(self._SYNC)
             sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
 
-            cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+            # get failsafe topology
+            failsafe = nodes.get(self._FAILSAFE)
+            try:
+                failsafe = json.loads(failsafe.value) if failsafe else None
+            except Exception:
+                failsafe = None
+
+            cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
         except etcd.EtcdKeyNotFound:
-            cluster = Cluster(None, None, None, None, [], None, None, None, None)
+            cluster = Cluster(None, None, None, None, [], None, None, None, None, None)
         except Exception as e:
             self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
         self._has_failed = False
@@ -716,6 +723,10 @@ class Etcd(AbstractEtcd):
         except etcd.EtcdKeyNotFound:
             return self._do_attempt_to_acquire_leader()
 
+    @catch_etcd_errors
+    def _write_failsafe(self, value):
+        return self._client.set(self.failsafe_path, value)
+
     @catch_return_false_exception
     def _update_leader(self):
         return self._run_and_handle_exceptions(self._do_update_leader, retry=None)
diff --git a/patroni/dcs/etcd3.py b/patroni/dcs/etcd3.py
index 3403f323..0c2d144b 100644
--- a/patroni/dcs/etcd3.py
+++ b/patroni/dcs/etcd3.py
@@ -712,7 +712,14 @@ class Etcd3(AbstractEtcd):
             sync = nodes.get(self._SYNC)
             sync = SyncState.from_node(sync and sync['mod_revision'], sync and sync['value'])
 
-            cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+            # get failsafe topology
+            failsafe = nodes.get(self._FAILSAFE)
+            try:
+                failsafe = json.loads(failsafe['value']) if failsafe else None
+            except Exception:
+                failsafe = None
+
+            cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
         except UnsupportedEtcdVersion:
             raise
         except Exception as e:
@@ -801,6 +808,10 @@ class Etcd3(AbstractEtcd):
     def _write_status(self, value):
         return self._client.put(self.status_path, value)
 
+    @catch_etcd_errors
+    def _write_failsafe(self, value):
+        return self._client.put(self.failsafe_path, value)
+
     @catch_return_false_exception
     def _update_leader(self):
         retry = self._retry.copy()
diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py
index a0fe3dfe..67541a34 100644
--- a/patroni/dcs/kubernetes.py
+++ b/patroni/dcs/kubernetes.py
@@ -832,6 +832,13 @@ class Kubernetes(AbstractDCS):
             except Exception:
                 slots = None
 
+            # get failsafe topology
+            failsafe = annotations.get(self._FAILSAFE)
+            try:
+                failsafe = json.loads(failsafe) if failsafe else None
+            except Exception:
+                failsafe = None
+
             # get leader
             leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
                              'ttl', 'renewTime', 'transitions') if n in annotations}
@@ -864,7 +871,7 @@ class Kubernetes(AbstractDCS):
             metadata = sync and sync.metadata
             sync = SyncState.from_node(metadata and metadata.resource_version,  metadata and metadata.annotations)
 
-            return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+            return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
         except Exception:
             logger.exception('get_cluster')
             raise KubernetesError('Kubernetes API is not responding properly')
@@ -999,6 +1006,9 @@ class Kubernetes(AbstractDCS):
     def _write_status(self, value):
         """Unused"""
 
+    def _write_failsafe(self, value):
+        """Unused"""
+
     def _update_leader(self):
         """Unused"""
 
@@ -1050,7 +1060,7 @@ class Kubernetes(AbstractDCS):
         return self._run_and_handle_exceptions(self._patch_or_create, self.leader_path, annotations,
                                                kind_resource_version, ips=ips, retry=_retry)
 
-    def update_leader(self, last_lsn, slots=None):
+    def update_leader(self, last_lsn, slots=None, failsafe=None):
         kind = self._kinds.get(self.leader_path)
         kind_annotations = kind and kind.metadata.annotations or {}
 
@@ -1064,7 +1074,10 @@ class Kubernetes(AbstractDCS):
                        'transitions': leader_observed_record.get('transitions') or '0'}
         if last_lsn:
             annotations[self._OPTIME] = str(last_lsn)
-            annotations['slots'] = json.dumps(slots) if slots else None
+            annotations['slots'] = json.dumps(slots, separators=(',', ':')) if slots else None
+
+        if failsafe is not None:
+            annotations[self._FAILSAFE] = json.dumps(failsafe, separators=(',', ':')) if failsafe else None
 
         resource_version = kind and kind.metadata.resource_version
         return self._update_leader_with_retry(annotations, resource_version, self.__ips)
diff --git a/patroni/dcs/raft.py b/patroni/dcs/raft.py
index 1318c188..8c563906 100644
--- a/patroni/dcs/raft.py
+++ b/patroni/dcs/raft.py
@@ -323,7 +323,7 @@ class Raft(AbstractDCS):
         prefix = self.client_path('')
         response = self._sync_obj.get(prefix, recursive=True)
         if not response:
-            return Cluster(None, None, None, None, [], None, None, None, None)
+            return Cluster(None, None, None, None, [], None, None, None, None, None)
         nodes = {os.path.relpath(key, prefix).replace('\\', '/'): value for key, value in response.items()}
 
         # get initialize flag
@@ -376,7 +376,14 @@ class Raft(AbstractDCS):
         sync = nodes.get(self._SYNC)
         sync = SyncState.from_node(sync and sync['index'], sync and sync['value'])
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+        # get failsafe topology
+        failsafe = nodes.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe['value']) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
 
     def _write_leader_optime(self, last_lsn):
         return self._sync_obj.set(self.leader_optime_path, last_lsn, timeout=1)
@@ -384,6 +391,9 @@ class Raft(AbstractDCS):
     def _write_status(self, value):
         return self._sync_obj.set(self.status_path, value, timeout=1)
 
+    def _write_failsafe(self, value):
+        return self._sync_obj.set(self.failsafe_path, value, timeout=1)
+
     def _update_leader(self):
         ret = self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl,
                                  handle_raft_error=False, prevValue=self._name)
diff --git a/patroni/dcs/zookeeper.py b/patroni/dcs/zookeeper.py
index 9101244f..c4f686a6 100644
--- a/patroni/dcs/zookeeper.py
+++ b/patroni/dcs/zookeeper.py
@@ -276,7 +276,14 @@ class ZooKeeper(AbstractDCS):
         failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
         failover = failover and Failover.from_node(failover[1].version, failover[0])
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+        # get failsafe topology
+        failsafe = self.get_node(self.failsafe_path, watch=self.cluster_watcher) if self._FAILSAFE in nodes else None
+        try:
+            failsafe = json.loads(failsafe[0]) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
 
     def _load_cluster(self):
         cluster = self.cluster
@@ -297,8 +304,8 @@ class ZooKeeper(AbstractDCS):
                 try:
                     last_lsn, slots = self.get_status(cluster.leader)
                     self.event.clear()
-                    cluster = Cluster(cluster.initialize, cluster.config, cluster.leader, last_lsn,
-                                      cluster.members, cluster.failover, cluster.sync, cluster.history, slots)
+                    cluster = Cluster(cluster.initialize, cluster.config, cluster.leader, last_lsn, cluster.members,
+                                      cluster.failover, cluster.sync, cluster.history, slots, cluster.failsafe)
                 except Exception:
                     pass
         return cluster
@@ -408,6 +415,9 @@ class ZooKeeper(AbstractDCS):
     def _write_status(self, value):
         return self._set_or_create(self.status_path, value)
 
+    def _write_failsafe(self, value):
+        return self._set_or_create(self.failsafe_path, value)
+
     def _update_leader(self):
         cluster = self.cluster
         session = cluster and isinstance(cluster.leader, Leader) and cluster.leader.session


Test patch:
diff --git a/tests/test_consul.py b/tests/test_consul.py
index 03c90273..caf99bb0 100644
--- a/tests/test_consul.py
+++ b/tests/test_consul.py
@@ -34,6 +34,8 @@ def kv_get(self, key, **kwargs):
                   'ModifyIndex': 6429, 'Value': b'4496294792'},
                  {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'sync', 'LockIndex': 0,
                   'ModifyIndex': 6429, 'Value': b'{"leader": "leader", "sync_standby": null}'},
+                 {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'failsafe', 'LockIndex': 0,
+                  'ModifyIndex': 6429, 'Value': b'{'},
                  {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'status', 'LockIndex': 0,
                   'ModifyIndex': 6429, 'Value': b'{"optime":4496294792, "slots":{"ls":12345}}'}])
     if key == 'service/good/':
@@ -167,7 +169,7 @@ class TestConsul(unittest.TestCase):
         self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8'
         with patch.object(consul.Consul.KV, 'delete', Mock(return_value=True)):
             with patch.object(consul.Consul.KV, 'put', Mock(return_value=True)):
-                self.assertTrue(self.c.update_leader(12345))
+                self.assertTrue(self.c.update_leader(12345, failsafe={'foo': 'bar'}))
             with patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException)):
                 self.assertFalse(self.c.update_leader(12345))
             with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 100, 200, 300])):
diff --git a/tests/test_etcd.py b/tests/test_etcd.py
index d22d9760..61980474 100644
--- a/tests/test_etcd.py
+++ b/tests/test_etcd.py
@@ -67,6 +67,7 @@ def etcd_read(self, key, **kwargs):
                      "expiration": "2015-05-15T09:11:09.611860899Z", "ttl": 30,
                      "modifiedIndex": 20730, "createdIndex": 20730}],
                  "modifiedIndex": 1581, "createdIndex": 1581},
+                {"key": "/service/batman5/failsafe", "value": '{', "modifiedIndex": 1582, "createdIndex": 1582},
                 {"key": "/service/batman5/status", "value": '{"optime":2164261704,"slots":{"ls":12345}}',
                  "modifiedIndex": 1582, "createdIndex": 1582}], "modifiedIndex": 1581, "createdIndex": 1581}}
     if key == '/service/legacy/':
@@ -286,7 +287,7 @@ class TestEtcd(unittest.TestCase):
         self.etcd.write_leader_optime('0')
 
     def test_update_leader(self):
-        self.assertTrue(self.etcd.update_leader(None))
+        self.assertTrue(self.etcd.update_leader(None, failsafe={'foo': 'bar'}))
         with patch.object(etcd.Client, 'write',
                           Mock(side_effect=[etcd.EtcdConnectionFailed, etcd.EtcdClusterIdChanged, Exception])):
             self.assertRaises(EtcdError, self.etcd.update_leader, None)
diff --git a/tests/test_etcd3.py b/tests/test_etcd3.py
index e09c59bc..a4ba5841 100644
--- a/tests/test_etcd3.py
+++ b/tests/test_etcd3.py
@@ -36,7 +36,8 @@ def mock_urlopen(self, method, url, **kwargs):
                  "value": base64_encode('{}'), "lease": "123", "mod_revision": '1'},
                 {"key": base64_encode('/patroni/test/members/bar'),
                  "value": base64_encode('{"version":"1.6.5"}'), "lease": "123", "mod_revision": '1'},
-                {"key": base64_encode('/patroni/test/failover'), "value": base64_encode('{}'), "mod_revision": '1'}
+                {"key": base64_encode('/patroni/test/failover'), "value": base64_encode('{}'), "mod_revision": '1'},
+                {"key": base64_encode('/patroni/test/failsafe'), "value": base64_encode('{'), "mod_revision": '1'}
             ]
         })
     elif url.endswith('/watch'):
@@ -215,7 +216,7 @@ class TestEtcd3(BaseTestEtcd3):
 
     def test__update_leader(self):
         self.etcd3._lease = None
-        self.etcd3.update_leader('123')
+        self.etcd3.update_leader('123', failsafe={'foo': 'bar'})
         self.etcd3._last_lease_refresh = 0
         self.etcd3.update_leader('124')
         with patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(return_value=True)),\
diff --git a/tests/test_ha.py b/tests/test_ha.py
index 7a7fb5f0..4ec0d7f7 100644
--- a/tests/test_ha.py
+++ b/tests/test_ha.py
@@ -38,7 +38,7 @@ def get_cluster(initialize, leader, members, failover, sync, cluster_config=None
     history = TimelineHistory(1, '[[1,67197376,"no recovery target specified","' + t + '","foo"]]',
                               [(1, 67197376, 'no recovery target specified', t, 'foo')])
     cluster_config = cluster_config or ClusterConfig(1, {'check_timeline': True}, 1)
-    return Cluster(initialize, cluster_config, leader, 10, members, failover, sync, history, None)
+    return Cluster(initialize, cluster_config, leader, 10, members, failover, sync, history, None, None)
 
 
 def get_cluster_not_initialized_without_leader(cluster_config=None):
diff --git a/tests/test_kubernetes.py b/tests/test_kubernetes.py
index 9ff376a9..2dd1fd70 100644
--- a/tests/test_kubernetes.py
+++ b/tests/test_kubernetes.py
@@ -19,7 +19,7 @@ def mock_list_namespaced_config_map(*args, **kwargs):
                 'annotations': {'initialize': '123', 'config': '{}'}}
     items = [k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))]
     metadata.update({'name': 'test-leader',
-                     'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{'}})
+                     'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{', 'failsafe': '{'}})
     items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
     metadata.update({'name': 'test-failover', 'annotations': {'leader': 'p-0'}})
     items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
@@ -300,7 +300,7 @@ class TestKubernetesEndpoints(BaseTestKubernetes):
 
     @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', create=True)
     def test_update_leader(self, mock_patch_namespaced_endpoints):
-        self.assertIsNotNone(self.k.update_leader('123'))
+        self.assertIsNotNone(self.k.update_leader('123', failsafe={'foo': 'bar'}))
         args = mock_patch_namespaced_endpoints.call_args[0]
         self.assertEqual(args[2].subsets[0].addresses[0].target_ref.resource_version, '10')
         self.k._kinds._object_cache['test'].subsets[:] = []
diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py
index 9ee2d9a2..df24ba55 100644
--- a/tests/test_postgresql.py
+++ b/tests/test_postgresql.py
@@ -640,7 +640,7 @@ class TestPostgresql(BaseTestPostgresql):
 
     def test_pick_sync_standby(self):
         cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None,
-                          SyncState(0, self.me.name, self.leadermem.name), None, None)
+                          SyncState(0, self.me.name, self.leadermem.name), None, None, None)
         mock_cursor = Mock()
         mock_cursor.fetchone.return_value = ('remote_apply',)
 
diff --git a/tests/test_raft.py b/tests/test_raft.py
index 9797ec0c..2c8ce84e 100644
--- a/tests/test_raft.py
+++ b/tests/test_raft.py
@@ -140,7 +140,8 @@ class TestRaft(unittest.TestCase):
         raft.get_cluster()
         self.assertTrue(raft._sync_obj.set(raft.status_path, '{"optime":1234567,"slots":{"ls":12345}}'))
         raft.get_cluster()
-        self.assertTrue(raft.update_leader('1'))
+        self.assertTrue(raft.update_leader('1', failsafe={'foo': 'bat'}))
+        self.assertTrue(raft._sync_obj.set(raft.failsafe_path, '{"foo"}'))
         self.assertTrue(raft._sync_obj.set(raft.status_path, '{'))
         raft.get_cluster()
         self.assertTrue(raft.delete_sync_state())
diff --git a/tests/test_slots.py b/tests/test_slots.py
index f96bc6d5..4b19fc4a 100644
--- a/tests/test_slots.py
+++ b/tests/test_slots.py
@@ -31,14 +31,14 @@ class TestSlotsHandler(BaseTestPostgresql):
         self.p.start()
         config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}}, 1)
         self.cluster = Cluster(True, config, self.leader, 0,
-                               [self.me, self.other, self.leadermem], None, None, None, {'ls': 12345})
+                               [self.me, self.other, self.leadermem], None, None, None, {'ls': 12345}, None)
 
     def test_sync_replication_slots(self):
         config = ClusterConfig(1, {'slots': {'test_3': {'database': 'a', 'plugin': 'b'},
                                              'A': 0, 'ls': 0, 'b': {'type': 'logical', 'plugin': '1'}},
                                    'ignore_slots': [{'name': 'blabla'}]}, 1)
         cluster = Cluster(True, config, self.leader, 0,
-                          [self.me, self.other, self.leadermem], None, None, None, {'test_3': 10})
+                          [self.me, self.other, self.leadermem], None, None, None, {'test_3': 10}, None)
         with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg.OperationalError)):
             self.s.sync_replication_slots(cluster, False)
         self.p.set_role('standby_leader')
@@ -67,7 +67,8 @@ class TestSlotsHandler(BaseTestPostgresql):
     def test_process_permanent_slots(self):
         config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}},
                                    'ignore_slots': [{'name': 'blabla'}]}, 1)
-        cluster = Cluster(True, config, self.leader, 0, [self.me, self.other, self.leadermem], None, None, None, None)
+        cluster = Cluster(True, config, self.leader, 0,
+                          [self.me, self.other, self.leadermem], None, None, None, None, None)
 
         self.s.sync_replication_slots(cluster, False)
         with patch.object(Postgresql, '_query') as mock_query:
diff --git a/tests/test_zookeeper.py b/tests/test_zookeeper.py
index effcf6de..e1e2ac66 100644
--- a/tests/test_zookeeper.py
+++ b/tests/test_zookeeper.py
@@ -51,6 +51,8 @@ class MockKazooClient(Mock):
             return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
         elif path.endswith('/status'):
             return (b'{"optime":500,"slots":{"ls":1234567}}', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
+        elif path.endswith('/failsafe'):
+            return (b'{a}', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
         return (b'', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
 
     @staticmethod
@@ -60,7 +62,7 @@ class MockKazooClient(Mock):
         if path.startswith('/no_node'):
             raise NoNodeError
         elif path in ['/service/bla/', '/service/test/']:
-            return ['initialize', 'leader', 'members', 'optime', 'failover', 'sync']
+            return ['initialize', 'leader', 'members', 'optime', 'failover', 'sync', 'failsafe']
         return ['foo', 'bar', 'buzz']
 
     def create(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
@@ -237,7 +239,7 @@ class TestZooKeeper(unittest.TestCase):
         with patch.object(MockKazooClient, 'delete', Mock(side_effect=RetryFailedError)):
             self.assertRaises(ZooKeeperError, self.zk.update_leader, 12345)
         with patch.object(MockKazooClient, 'delete', Mock(side_effect=NoNodeError)):
-            self.assertTrue(self.zk.update_leader(12345))
+            self.assertTrue(self.zk.update_leader(12345, failsafe={'foo': 'bar'}))
             with patch.object(MockKazooClient, 'create', Mock(side_effect=[RetryFailedError, Exception])):
                 self.assertRaises(ZooKeeperError, self.zk.update_leader, 12345)
                 self.assertFalse(self.zk.update_leader(12345))