Repository: zalando/patroni
Pull request #2833 (2023-08-24 14:22:12)

Introduce a dedicated postgres connection for REST API
Sharing a single connection between REST API and the main thread (doing heartbeats) was working mostly fine, except when Postgres becomes so slow that REST API queries start blocking the main loop.

If the dedicated REST API connection isn't available we use the heartbeat connection as a fallback.

coveralls: ## Pull Request Test Coverage Report for [Build 5965009580](https://coveralls.io/builds/62205780)

* **10** of **10**   **(100.0%)**  changed or added relevant lines in **1** file are covered.
* No unchanged relevant lines lost coverage.
* Overall coverage remained the same at **99.862%**

---



|  Totals | [![Coverage Status](https://coveralls.io/builds/62205780/badge)](https://coveralls.io/builds/62205780) |
| :-- | --: |
| Change from base [Build 5964899763](https://coveralls.io/builds/62205471): |  0.0% |
| Covered Lines: | 13068 |
| Relevant Lines: | 13086 |

---
##### 💛  - [Coveralls](https://coveralls.io)


Patch:
diff --git a/.gitignore b/.gitignore
index 23cbf17a..c902c6eb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,7 +33,7 @@ nosetests.xml
 coverage.xml
 htmlcov
 junit.xml
-features/output
+features/output*
 dummy
 
 # Translations
diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst
index 00595cf7..e3762aca 100644
--- a/docs/ENVIRONMENT.rst
+++ b/docs/ENVIRONMENT.rst
@@ -115,7 +115,7 @@ Kubernetes
 -  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing role (master or replica or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``.
 -  **PATRONI\_KUBERNETES\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `master`. Default value is `master`.
 -  **PATRONI\_KUBERNETES\_FOLLOWER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `replica`. Default value is `replica`.
--  **PATRONI\_KUBERNETES\_STANDBY\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is ``standby-leader``. Default value is ``standby-leader``.
+-  **PATRONI\_KUBERNETES\_STANDBY\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is ``standby_leader``. Default value is ``master``.
 -  **PATRONI\_KUBERNETES\_TMP\_ROLE\_LABEL**: (optional) name of the temporary label containing role (master or replica). Value of this label will always use the default of corresponding role. Set only when necessary.
 -  **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
diff --git a/docs/rest_api.rst b/docs/rest_api.rst
index 6e97ade0..e49f6ee0 100644
--- a/docs/rest_api.rst
+++ b/docs/rest_api.rst
@@ -131,7 +131,8 @@ The ``GET /patroni`` is used by Patroni during the leader race. It also could be
       "database_system_identifier": "7268616322854375442",
       "patroni": {
         "version": "3.1.0",
-        "scope": "demo"
+        "scope": "demo",
+        "name": "patroni1"
       }
     }
 
@@ -178,7 +179,8 @@ The ``GET /patroni`` is used by Patroni during the leader race. It also could be
       "database_system_identifier": "7268616322854375442",
       "patroni": {
         "version": "3.1.0",
-        "scope": "demo"
+        "scope": "demo",
+        "name": "patroni1"
       }
     }
 
@@ -223,7 +225,8 @@ The ``GET /patroni`` is used by Patroni during the leader race. It also could be
       "database_system_identifier": "7268616322854375442",
       "patroni": {
         "version": "3.1.0",
-        "scope": "demo"
+        "scope": "demo",
+        "name": "patroni1"
       }
     }
 
@@ -267,7 +270,8 @@ The ``GET /patroni`` is used by Patroni during the leader race. It also could be
       "database_system_identifier": "7268616322854375442",
       "patroni": {
         "version": "3.1.0",
-        "scope": "demo"
+        "scope": "demo",
+        "name": "patroni1"
       }
     }
 
@@ -279,70 +283,70 @@ Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` e
 	
 	# HELP patroni_version Patroni semver without periods. \
 	# TYPE patroni_version gauge
-	patroni_version{scope="batman"} 020103
+	patroni_version{scope="batman",name="patroni1"} 020103
 	# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.
 	# TYPE patroni_postgres_running gauge
-	patroni_postgres_running{scope="batman"} 1
+	patroni_postgres_running{scope="batman",name="patroni1"} 1
 	# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.
 	# TYPE patroni_postmaster_start_time gauge
-	patroni_postmaster_start_time{scope="batman"} 1657656955.179243
+	patroni_postmaster_start_time{scope="batman",name="patroni1"} 1657656955.179243
 	# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.
 	# TYPE patroni_master gauge
-	patroni_master{scope="batman"} 1
+	patroni_master{scope="batman",name="patroni1"} 1
 	# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.
 	# TYPE patroni_primary gauge
-	patroni_primary{scope="batman"} 1
+	patroni_primary{scope="batman",name="patroni1"} 1
 	# HELP patroni_xlog_location Current location of the Postgres transaction log, 0 if this node is not the leader.
 	# TYPE patroni_xlog_location counter
-	patroni_xlog_location{scope="batman"} 22320573386952
+	patroni_xlog_location{scope="batman",name="patroni1"} 22320573386952
 	# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.
 	# TYPE patroni_standby_leader gauge
-	patroni_standby_leader{scope="batman"} 0
+	patroni_standby_leader{scope="batman",name="patroni1"} 0
 	# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.
 	# TYPE patroni_replica gauge
-	patroni_replica{scope="batman"} 0
+	patroni_replica{scope="batman",name="patroni1"} 0
 	# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.
 	# TYPE patroni_sync_standby gauge
-	patroni_sync_standby{scope="batman"} 0
+	patroni_sync_standby{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_received_location Current location of the received Postgres transaction log, 0 if this node is not a replica.
 	# TYPE patroni_xlog_received_location counter
-	patroni_xlog_received_location{scope="batman"} 0
+	patroni_xlog_received_location{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_replayed_location Current location of the replayed Postgres transaction log, 0 if this node is not a replica.
 	# TYPE patroni_xlog_replayed_location counter
-	patroni_xlog_replayed_location{scope="batman"} 0
+	patroni_xlog_replayed_location{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed Postgres transaction log, 0 if null.
 	# TYPE patroni_xlog_replayed_timestamp gauge
-	patroni_xlog_replayed_timestamp{scope="batman"} 0
+	patroni_xlog_replayed_timestamp{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.
 	# TYPE patroni_xlog_paused gauge
-	patroni_xlog_paused{scope="batman"} 0
+	patroni_xlog_paused{scope="batman",name="patroni1"} 0
 	# HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise.
 	# TYPE patroni_postgres_streaming gauge
-	patroni_postgres_streaming{scope="batman"} 1
+	patroni_postgres_streaming{scope="batman",name="patroni1"} 1
 	# HELP patroni_postgres_in_archive_recovery Value is 1 if Postgres is replicating from archive, 0 otherwise.
 	# TYPE patroni_postgres_in_archive_recovery gauge
-	patroni_postgres_in_archive_recovery{scope="batman"} 0
+	patroni_postgres_in_archive_recovery{scope="batman",name="patroni1"} 0
 	# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.
 	# TYPE patroni_postgres_server_version gauge
-	patroni_postgres_server_version {scope="batman"} 140004
+	patroni_postgres_server_version{scope="batman",name="patroni1"} 140004
 	# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.
 	# TYPE patroni_cluster_unlocked gauge
-	patroni_cluster_unlocked{scope="batman"} 0
+	patroni_cluster_unlocked{scope="batman",name="patroni1"} 0
 	# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.
 	# TYPE patroni_postgres_timeline counter
-	patroni_failsafe_mode_is_active{scope="batman"} 0
+	patroni_failsafe_mode_is_active{scope="batman",name="patroni1"} 0
 	# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.
 	# TYPE patroni_postgres_timeline counter
-	patroni_postgres_timeline{scope="batman"} 24
+	patroni_postgres_timeline{scope="batman",name="patroni1"} 24
 	# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully by Patroni.
 	# TYPE patroni_dcs_last_seen gauge
-	patroni_dcs_last_seen{scope="batman"} 1677658321
+	patroni_dcs_last_seen{scope="batman",name="patroni1"} 1677658321
 	# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.
 	# TYPE patroni_pending_restart gauge
-	patroni_pending_restart{scope="batman"} 1
+	patroni_pending_restart{scope="batman",name="patroni1"} 1
 	# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.
 	# TYPE patroni_is_paused gauge
-	patroni_is_paused{scope="batman"} 1
+	patroni_is_paused{scope="batman",name="patroni1"} 1
 
 
 Cluster status endpoints
@@ -381,6 +385,7 @@ Cluster status endpoints
           "lag": 0
         }
       ],
+      "scope": "demo",
       "scheduled_switchover": {
         "at": "2023-09-24T10:36:00+02:00",
         "from": "patroni1",
@@ -489,8 +494,9 @@ Let's check that the node processed this configuration. First of all it should s
 	    "location": 2197818976
 	  },
 	  "patroni": {
+	    "version": "1.0",
 	    "scope": "batman",
-	    "version": "1.0"
+	    "name": "patroni1"
 	  },
 	  "state": "running",
 	  "role": "master",
diff --git a/docs/yaml_configuration.rst b/docs/yaml_configuration.rst
index f0cb8ef4..d9283192 100644
--- a/docs/yaml_configuration.rst
+++ b/docs/yaml_configuration.rst
@@ -171,7 +171,7 @@ Kubernetes
 -  **role\_label**: (optional) name of the label containing role (master or replica or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``.
 -  **leader\_label\_value**: (optional) value of the pod label when Postgres role is ``master``. Default value is ``master``.
 -  **follower\_label\_value**: (optional) value of the pod label when Postgres role is ``replica``. Default value is ``replica``.
--  **standby\_leader\_label\_value**: (optional) value of the pod label when Postgres role is ``standby-leader``. Default value is ``standby-leader``.
+-  **standby\_leader\_label\_value**: (optional) value of the pod label when Postgres role is ``standby_leader``. Default value is ``master``.
 -  **tmp_\role\_label**: (optional) name of the temporary label containing role (master or replica). Value of this label will always use the default of corresponding role. Set only when necessary.
 -  **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
diff --git a/features/Dockerfile b/features/Dockerfile
index 86c6041d..7f52c2af 100644
--- a/features/Dockerfile
+++ b/features/Dockerfile
@@ -27,8 +27,8 @@ RUN set -ex \
     && apt-get update \
     && apt-get reinstall init-system-helpers \
     && apt-get install -y \
-      python3-pip \
       python3-dev \
+      python3-venv \
       rsync \
       curl \
       gcc \
@@ -40,7 +40,9 @@ RUN set -ex \
       net-tools \
       iputils-ping \
     && rm -rf /var/cache/apt \
-    && python3 -m pip install --no-cache-dir tox \
+    \
+    && python3 -m venv /tox \
+    && /tox/bin/pip install --no-cache-dir tox>=4 \
     \
     && mkdir -p "$PGHOME" \
     && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
@@ -50,6 +52,7 @@ RUN set -ex \
     && curl -sL "$ETCDURL/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \
        | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
 
+ENV PATH="/tox/bin:$PATH"
 
 # This Dockerfile syntax only works with docker buildx and the syntax
 # line at the top of this file.
diff --git a/patroni/api.py b/patroni/api.py
index b01c24d8..4647e7c2 100644
--- a/patroni/api.py
+++ b/patroni/api.py
@@ -198,7 +198,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response['database_system_identifier'] = patroni.postgresql.sysid
         if patroni.postgresql.pending_restart:
             response['pending_restart'] = True
-        response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
+        response['patroni'] = {
+            'version': patroni.version,
+            'scope': patroni.postgresql.scope,
+            'name': patroni.postgresql.name
+        }
         if patroni.scheduled_restart:
             response['scheduled_restart'] = patroni.scheduled_restart.copy()
             del response['scheduled_restart']['postmaster_start_time']
@@ -449,7 +453,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """
         cluster = self.server.patroni.dcs.get_cluster(True)
         global_config = self.server.patroni.config.get_global_config(cluster)
-        self._write_json_response(200, cluster_as_json(cluster, global_config))
+
+        response = cluster_as_json(cluster, global_config)
+        response['scope'] = self.server.patroni.postgresql.scope
+        self._write_json_response(200, response)
 
     def do_GET_history(self) -> None:
         """Handle a ``GET`` request to ``/history`` path.
@@ -526,113 +533,113 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         metrics: List[str] = []
 
-        scope_label = '{{scope="{0}"}}'.format(patroni.postgresql.scope)
+        labels = f'{{scope="{patroni.postgresql.scope}",name="{patroni.postgresql.name}"}}'
         metrics.append("# HELP patroni_version Patroni semver without periods.")
         metrics.append("# TYPE patroni_version gauge")
         padded_semver = ''.join([x.zfill(2) for x in patroni.version.split('.')])  # 2.0.2 => 020002
-        metrics.append("patroni_version{0} {1}".format(scope_label, padded_semver))
+        metrics.append("patroni_version{0} {1}".format(labels, padded_semver))
 
         metrics.append("# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_running gauge")
-        metrics.append("patroni_postgres_running{0} {1}".format(scope_label, int(postgres['state'] == 'running')))
+        metrics.append("patroni_postgres_running{0} {1}".format(labels, int(postgres['state'] == 'running')))
 
         metrics.append("# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.")
         metrics.append("# TYPE patroni_postmaster_start_time gauge")
         postmaster_start_time = postgres.get('postmaster_start_time')
         postmaster_start_time = (postmaster_start_time - epoch).total_seconds() if postmaster_start_time else 0
-        metrics.append("patroni_postmaster_start_time{0} {1}".format(scope_label, postmaster_start_time))
+        metrics.append("patroni_postmaster_start_time{0} {1}".format(labels, postmaster_start_time))
 
         metrics.append("# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.")
         metrics.append("# TYPE patroni_master gauge")
-        metrics.append("patroni_master{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
+        metrics.append("patroni_master{0} {1}".format(labels, int(postgres['role'] in ('master', 'primary'))))
 
         metrics.append("# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.")
         metrics.append("# TYPE patroni_primary gauge")
-        metrics.append("patroni_primary{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
+        metrics.append("patroni_primary{0} {1}".format(labels, int(postgres['role'] in ('master', 'primary'))))
 
         metrics.append("# HELP patroni_xlog_location Current location of the Postgres"
                        " transaction log, 0 if this node is not the leader.")
         metrics.append("# TYPE patroni_xlog_location counter")
-        metrics.append("patroni_xlog_location{0} {1}".format(scope_label, postgres.get('xlog', {}).get('location', 0)))
+        metrics.append("patroni_xlog_location{0} {1}".format(labels, postgres.get('xlog', {}).get('location', 0)))
 
         metrics.append("# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.")
         metrics.append("# TYPE patroni_standby_leader gauge")
-        metrics.append("patroni_standby_leader{0} {1}".format(scope_label, int(postgres['role'] == 'standby_leader')))
+        metrics.append("patroni_standby_leader{0} {1}".format(labels, int(postgres['role'] == 'standby_leader')))
 
         metrics.append("# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.")
         metrics.append("# TYPE patroni_replica gauge")
-        metrics.append("patroni_replica{0} {1}".format(scope_label, int(postgres['role'] == 'replica')))
+        metrics.append("patroni_replica{0} {1}".format(labels, int(postgres['role'] == 'replica')))
 
         metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.")
         metrics.append("# TYPE patroni_sync_standby gauge")
-        metrics.append("patroni_sync_standby{0} {1}".format(scope_label, int(postgres.get('sync_standby', False))))
+        metrics.append("patroni_sync_standby{0} {1}".format(labels, int(postgres.get('sync_standby', False))))
 
         metrics.append("# HELP patroni_xlog_received_location Current location of the received"
                        " Postgres transaction log, 0 if this node is not a replica.")
         metrics.append("# TYPE patroni_xlog_received_location counter")
         metrics.append("patroni_xlog_received_location{0} {1}"
-                       .format(scope_label, postgres.get('xlog', {}).get('received_location', 0)))
+                       .format(labels, postgres.get('xlog', {}).get('received_location', 0)))
 
         metrics.append("# HELP patroni_xlog_replayed_location Current location of the replayed"
                        " Postgres transaction log, 0 if this node is not a replica.")
         metrics.append("# TYPE patroni_xlog_replayed_location counter")
         metrics.append("patroni_xlog_replayed_location{0} {1}"
-                       .format(scope_label, postgres.get('xlog', {}).get('replayed_location', 0)))
+                       .format(labels, postgres.get('xlog', {}).get('replayed_location', 0)))
 
         metrics.append("# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed"
                        " Postgres transaction log, 0 if null.")
         metrics.append("# TYPE patroni_xlog_replayed_timestamp gauge")
         replayed_timestamp = postgres.get('xlog', {}).get('replayed_timestamp')
         replayed_timestamp = (replayed_timestamp - epoch).total_seconds() if replayed_timestamp else 0
-        metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(scope_label, replayed_timestamp))
+        metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(labels, replayed_timestamp))
 
         metrics.append("# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.")
         metrics.append("# TYPE patroni_xlog_paused gauge")
         metrics.append("patroni_xlog_paused{0} {1}"
-                       .format(scope_label, int(postgres.get('xlog', {}).get('paused', False) is True)))
+                       .format(labels, int(postgres.get('xlog', {}).get('paused', False) is True)))
 
         if postgres.get('server_version', 0) >= 90600:
             metrics.append("# HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise.")
             metrics.append("# TYPE patroni_postgres_streaming gauge")
             metrics.append("patroni_postgres_streaming{0} {1}"
-                           .format(scope_label, int(postgres.get('replication_state') == 'streaming')))
+                           .format(labels, int(postgres.get('replication_state') == 'streaming')))
 
             metrics.append("# HELP patroni_postgres_in_archive_recovery Value is 1"
                            " if Postgres is replicating from archive, 0 otherwise.")
             metrics.append("# TYPE patroni_postgres_in_archive_recovery gauge")
             metrics.append("patroni_postgres_in_archive_recovery{0} {1}"
-                           .format(scope_label, int(postgres.get('replication_state') == 'in archive recovery')))
+                           .format(labels, int(postgres.get('replication_state') == 'in archive recovery')))
 
         metrics.append("# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_server_version gauge")
-        metrics.append("patroni_postgres_server_version {0} {1}".format(scope_label, postgres.get('server_version', 0)))
+        metrics.append("patroni_postgres_server_version {0} {1}".format(labels, postgres.get('server_version', 0)))
 
         metrics.append("# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.")
         metrics.append("# TYPE patroni_cluster_unlocked gauge")
-        metrics.append("patroni_cluster_unlocked{0} {1}".format(scope_label, int(postgres.get('cluster_unlocked', 0))))
+        metrics.append("patroni_cluster_unlocked{0} {1}".format(labels, int(postgres.get('cluster_unlocked', 0))))
 
         metrics.append("# HELP patroni_failsafe_mode_is_active Value is 1 if failsafe mode is active, 0 if inactive.")
         metrics.append("# TYPE patroni_failsafe_mode_is_active gauge")
         metrics.append("patroni_failsafe_mode_is_active{0} {1}"
-                       .format(scope_label, int(postgres.get('failsafe_mode_is_active', 0))))
+                       .format(labels, int(postgres.get('failsafe_mode_is_active', 0))))
 
         metrics.append("# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_timeline counter")
-        metrics.append("patroni_postgres_timeline{0} {1}".format(scope_label, postgres.get('timeline', 0)))
+        metrics.append("patroni_postgres_timeline{0} {1}".format(labels, postgres.get('timeline', 0)))
 
         metrics.append("# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully"
                        " by Patroni.")
         metrics.append("# TYPE patroni_dcs_last_seen gauge")
-        metrics.append("patroni_dcs_last_seen{0} {1}".format(scope_label, postgres.get('dcs_last_seen', 0)))
+        metrics.append("patroni_dcs_last_seen{0} {1}".format(labels, postgres.get('dcs_last_seen', 0)))
 
         metrics.append("# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.")
         metrics.append("# TYPE patroni_pending_restart gauge")
         metrics.append("patroni_pending_restart{0} {1}"
-                       .format(scope_label, int(patroni.postgresql.pending_restart)))
+                       .format(labels, int(patroni.postgresql.pending_restart)))
 
         metrics.append("# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.")
         metrics.append("# TYPE patroni_is_paused gauge")
-        metrics.append("patroni_is_paused{0} {1}".format(scope_label, int(postgres.get('pause', 0))))
+        metrics.append("patroni_is_paused{0} {1}".format(labels, int(postgres.get('pause', 0))))
 
         self.write_response(200, '\n'.join(metrics) + '\n', content_type='text/plain')
 
@@ -1368,6 +1375,9 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
         """Execute *sql* query with *params* and optionally return results.
 
+        .. note::
+            Prefer to use own connection to postgres and fallback to ``heartbeat`` when own isn't available.
+
         :param sql: the SQL statement to be run.
         :param params: positional arguments to be used as parameters for *sql*.
 
@@ -1377,10 +1387,21 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
             :class:`psycopg.Error`: if had issues while executing *sql*.
             :class:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
         """
+        # We first try to get a heartbeat connection because it is always required for the main thread.
         try:
-            return self.patroni.postgresql.query(sql, *params, retry=False)
-        except RetryFailedError as e:
-            raise PostgresConnectionException(str(e))
+            heartbeat_connection = self.patroni.postgresql.connection_pool.get('heartbeat')
+            heartbeat_connection.get()  # try to open psycopg connection to postgres
+        except psycopg.Error as exc:
+            raise PostgresConnectionException('connection problems') from exc
+
+        try:
+            connection = self.patroni.postgresql.connection_pool.get('restapi')
+            connection.get()  # try to open psycopg connection to postgres
+        except psycopg.Error:
+            logger.debug('restapi connection to postgres is not available')
+            connection = heartbeat_connection
+
+        return connection.query(sql, *params)
 
     @staticmethod
     def _set_fd_cloexec(fd: socket.socket) -> None:
diff --git a/patroni/ctl.py b/patroni/ctl.py
index b1d63d2b..c8a396c9 100644
--- a/patroni/ctl.py
+++ b/patroni/ctl.py
@@ -264,7 +264,9 @@ role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 's
 @click.option('-k', '--insecure', is_flag=True, help='Allow connections to SSL sites without certs')
 @click.pass_context
 def ctl(ctx: click.Context, config_file: str, dcs_url: Optional[str], insecure: bool) -> None:
-    """Entry point of ``patronictl`` utility.
+    """Command-line interface for interacting with Patroni.
+    \f
+    Entry point of ``patronictl`` utility.
 
     Load the configuration file.
 
diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py
index fec544b0..47ba4e7a 100644
--- a/patroni/dcs/kubernetes.py
+++ b/patroni/dcs/kubernetes.py
@@ -756,7 +756,7 @@ class Kubernetes(AbstractDCS):
         self._role_label = config.get('role_label', 'role')
         self._leader_label_value = config.get('leader_label_value', 'master')
         self._follower_label_value = config.get('follower_label_value', 'replica')
-        self._standby_leader_label_value = config.get('standby_leader_label_value', 'standby-leader')
+        self._standby_leader_label_value = config.get('standby_leader_label_value', 'master')
         self._tmp_role_label = config.get('tmp_role_label')
         self._ca_certs = os.environ.get('PATRONI_KUBERNETES_CACERT', config.get('cacert')) or SERVICE_CERT_FILENAME
         super(Kubernetes, self).__init__({**config, 'namespace': ''})
@@ -1269,13 +1269,10 @@ class Kubernetes(AbstractDCS):
     def touch_member(self, data: Dict[str, Any]) -> bool:
         cluster = self.cluster
         if cluster and cluster.leader and cluster.leader.name == self._name:
-            role = self._leader_label_value
+            role = self._standby_leader_label_value if data['role'] == 'standby_leader' else self._leader_label_value
             tmp_role = 'master'
         elif data['state'] == 'running' and data['role'] not in ('master', 'primary'):
-            role = {
-                'replica': self._follower_label_value,
-                'standby-leader': self._standby_leader_label_value,
-            }.get(data['role'], data['role'])
+            role = {'replica': self._follower_label_value}.get(data['role'], data['role'])
             tmp_role = data['role']
         else:
             role = None
diff --git a/requirements.docs.txt b/requirements.docs.txt
index 1a71f097..afb47378 100644
--- a/requirements.docs.txt
+++ b/requirements.docs.txt
@@ -1,5 +1,4 @@
 sphinx>=4
 sphinx_rtd_theme>1
 sphinxcontrib-apidoc
-sphinx-github-style
-pyyaml
+sphinx-github-style<1.0.3


Test patch:
diff --git a/tests/test_api.py b/tests/test_api.py
index f433ca18..fa9a6280 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -11,9 +11,12 @@ from socketserver import ThreadingMixIn
 from patroni.api import RestApiHandler, RestApiServer
 from patroni.config import GlobalConfig
 from patroni.dcs import ClusterConfig, Member
+from patroni.exceptions import PostgresConnectionException
 from patroni.ha import _MemberStatus
+from patroni.psycopg import OperationalError
 from patroni.utils import RetryFailedError, tzutc
 
+from . import MockConnect, psycopg_connect
 from .test_ha import get_cluster_initialized_without_leader
 
 
@@ -21,8 +24,29 @@ future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
 postmaster_start_time = datetime.datetime.now(tzutc)
 
 
-class MockPostgresql(object):
+class MockConnection:
 
+    @staticmethod
+    def get(*args):
+        return psycopg_connect()
+
+    @staticmethod
+    def query(sql, *params):
+        return [(postmaster_start_time, 0, '', 0, '', False, postmaster_start_time, 'streaming', None,
+                 '[{"application_name":"walreceiver","client_addr":"1.2.3.4",'
+                 + '"state":"streaming","sync_state":"async","sync_priority":0}]')]
+
+
+class MockConnectionPool:
+
+    @staticmethod
+    def get(*args):
+        return MockConnection()
+
+
+class MockPostgresql:
+
+    connection_pool = MockConnectionPool()
     name = 'test'
     state = 'running'
     role = 'primary'
@@ -54,12 +78,6 @@ class MockPostgresql(object):
     def replication_state_from_parameters(*args):
         return 'streaming'
 
-    @staticmethod
-    def query(sql, *params, retry=False):
-        return [(postmaster_start_time, 0, '', 0, '', False, postmaster_start_time, 'streaming', None,
-                 '[{"application_name":"walreceiver","client_addr":"1.2.3.4",'
-                 + '"state":"streaming","sync_state":"async","sync_priority":0}]')]
-
 
 class MockWatchdog(object):
     is_healthy = False
@@ -487,7 +505,7 @@ class TestRestApiHandler(unittest.TestCase):
 
     @patch('time.sleep', Mock())
     def test_RestApiServer_query(self):
-        with patch.object(MockPostgresql, 'query', Mock(side_effect=RetryFailedError('bla'))):
+        with patch.object(MockConnection, 'query', Mock(side_effect=RetryFailedError('bla'))):
             self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
     @patch('time.sleep', Mock())
@@ -659,3 +677,11 @@ class TestRestApiServer(unittest.TestCase):
 
     def test_get_certificate_serial_number(self):
         self.assertIsNone(self.srv.get_certificate_serial_number())
+
+    def test_query(self):
+        with patch.object(MockConnection, 'get', Mock(side_effect=OperationalError)):
+            self.assertRaises(PostgresConnectionException, self.srv.query, 'SELECT 1')
+        with patch.object(MockConnection, 'get', Mock(side_effect=[MockConnect(), OperationalError])), \
+                patch.object(MockConnection, 'query') as mock_query:
+            self.srv.query('SELECT 1')
+            mock_query.assert_called_once_with('SELECT 1')
diff --git a/tests/test_kubernetes.py b/tests/test_kubernetes.py
index 694cd505..c33bbecc 100644
--- a/tests/test_kubernetes.py
+++ b/tests/test_kubernetes.py
@@ -308,13 +308,15 @@ class TestKubernetesConfigMaps(BaseTestKubernetes):
         mock_patch_namespaced_pod.assert_called()
         self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false')
         self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'replica')
+        mock_patch_namespaced_pod.rest_mock()
 
-        self.k.touch_member({'state': 'running', 'role': 'standby-leader'})
+        self.k._name = 'p-0'
+        self.k.touch_member({'role': 'standby_leader'})
         mock_patch_namespaced_pod.assert_called()
         self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false')
-        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'standby-leader')
+        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'master')
+        mock_patch_namespaced_pod.rest_mock()
 
-        self.k._name = 'p-0'
         self.k.touch_member({'role': 'primary'})
         mock_patch_namespaced_pod.assert_called()
         self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'true')