Repository: zalando/patroni
Pull request #2395 (2022-08-24 14:41:33)

Improve liveness probe
it will start failing if the heartbeat loop isn't running longer than `ttl` on the primary or `2*ttl` on the replica.

Close https://github.com/zalando/patroni/issues/2388

coveralls: ## Pull Request Test Coverage Report for [Build 2920012850](https://coveralls.io/builds/51911904)

* **5** of **5**   **(100.0%)**  changed or added relevant lines in **1** file are covered.
* No unchanged relevant lines lost coverage.
* Overall coverage increased (+4.0e**-05%**) to **99.883%**

---



|  Totals | [![Coverage Status](https://coveralls.io/builds/51911904/badge)](https://coveralls.io/builds/51911904) |
| :-- | --: |
| Change from base [Build 2918808690](https://coveralls.io/builds/51906420): |  4.0e-05% |
| Covered Lines: | 11123 |
| Relevant Lines: | 11136 |

---
##### 💛  - [Coveralls](https://coveralls.io)


Patch:
diff --git a/docs/rest_api.rst b/docs/rest_api.rst
index 5a2879d3..e6f9ecb3 100644
--- a/docs/rest_api.rst
+++ b/docs/rest_api.rst
@@ -58,7 +58,7 @@ For all health check ``GET`` requests Patroni returns a JSON document with the s
 
 - ``GET /health``: returns HTTP status code **200** only when PostgreSQL is up and running.
 
-- ``GET /liveness``: always returns HTTP status code **200** what only indicates that Patroni is running. Could be used for ``livenessProbe``.
+- ``GET /liveness``: returns HTTP status code **200** if Patroni heartbeat loop is properly running and **503** if the last run was more than ``ttl`` seconds ago on the primary or ``2*ttl`` on the replica. Could be used for ``livenessProbe``.
 
 - ``GET /readiness``: returns HTTP status code **200** when the Patroni node is running as the leader or when PostgreSQL is up and running. The endpoint could be used for ``readinessProbe`` when it is not possible to use Kubernetes endpoints for leader elections (OpenShift).
 
diff --git a/patroni/api.py b/patroni/api.py
index 963be433..d718a6e9 100644
--- a/patroni/api.py
+++ b/patroni/api.py
@@ -190,7 +190,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self.do_GET(write_status_code_only=True)
 
     def do_GET_liveness(self):
-        self._write_status_code_only(200)
+        patroni = self.server.patroni
+        is_primary = patroni.postgresql.role == 'master' and patroni.postgresql.is_running()
+        # We can tolerate Patroni problems longer on the replica.
+        # On the primary the liveness probe most likely will start failing only after the leader key expired.
+        # It should not be a big problem because replicas will see that the primary is still alive via REST API call.
+        liveness_threshold = patroni.dcs.ttl * (1 if is_primary else 2)
+
+        # In maintenance mode (pause) we are fine if heartbeat loop stuck.
+        status_code = 200 if patroni.ha.is_paused() or patroni.next_run + liveness_threshold > time.time() else 503
+        self._write_status_code_only(status_code)
 
     def do_GET_readiness(self):
         patroni = self.server.patroni
diff --git a/patroni/validator.py b/patroni/validator.py
index 595d4355..839a9016 100644
--- a/patroni/validator.py
+++ b/patroni/validator.py
@@ -37,6 +37,10 @@ def validate_host_port(host_port, listen=False, multiple_hosts=False):
             hosts = hosts.split(",")
         else:
             hosts = [hosts]
+        if "*" in hosts:
+            if len(hosts) != 1:
+                raise ConfigParseError("expecting '*' alone")
+            hosts = [p[-1][0] for p in socket.getaddrinfo(None, port, 0, socket.SOCK_STREAM, 0, socket.AI_PASSIVE)]
         for host in hosts:
             proto = socket.getaddrinfo(host, "", 0, socket.SOCK_STREAM, 0, socket.AI_PASSIVE)
             s = socket.socket(proto[0][0], socket.SOCK_STREAM)


Test patch:
diff --git a/tests/test_api.py b/tests/test_api.py
index 681814b0..73f81412 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -46,6 +46,10 @@ class MockPostgresql(object):
     def replica_cached_timeline(_):
         return 2
 
+    @staticmethod
+    def is_running():
+        return True
+
 
 class MockWatchdog(object):
     is_healthy = False
@@ -289,7 +293,9 @@ class TestRestApiHandler(unittest.TestCase):
     def test_do_HEAD(self):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'HEAD / HTTP/1.0'))
 
-    def test_do_GET_liveness(self):
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_GET_liveness(self, mock_dcs):
+        mock_dcs.ttl.return_value = PropertyMock(30)
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /liveness HTTP/1.0'))
 
     def test_do_GET_readiness(self):
diff --git a/tests/test_validator.py b/tests/test_validator.py
index 830b415e..9ccc6a7d 100644
--- a/tests/test_validator.py
+++ b/tests/test_validator.py
@@ -156,6 +156,7 @@ class TestValidator(unittest.TestCase):
         files.append(config["postgresql"]["bin_dir"])
         c = copy.deepcopy(config)
         c["restapi"]["connect_address"] = 'False:blabla'
+        c["postgresql"]["listen"] = '*:543'
         c["etcd"]["hosts"] = ["127.0.0.1:2379", "1244.0.0.1:2379", "127.0.0.1:invalidport"]
         c["kubernetes"]["pod_ip"] = "127.0.0.1111"
         errors = schema(c)
@@ -208,11 +209,12 @@ class TestValidator(unittest.TestCase):
         files.append(os.path.join(config["postgresql"]["data_dir"], "PG_VERSION"))
         c = copy.deepcopy(config)
         c["etcd"]["hosts"] = []
+        c["postgresql"]["listen"] = '127.0.0.2,*:543'
         del c["postgresql"]["bin_dir"]
         with patch('patroni.validator.open', mock_open(read_data='11')):
             errors = schema(c)
         output = "\n".join(errors)
-        self.assertEqual(['etcd.hosts', 'postgresql.data_dir',
+        self.assertEqual(['etcd.hosts', 'postgresql.data_dir', 'postgresql.listen',
                           'raft.bind_addr', 'raft.self_addr'], parse_output(output))
 
     @patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 12.1"))