Repository: red-hat-storage/ocs-ci
Pull request #12084: Update the noobaa_db_backup_and_recovery_locally fixture to use the new noobaa-db HA solution
Author: login: sagihirshfeld
Labels: name: Verified
description: Mark when PR was verified and log provided
name: size/L
description: PR that changes 100-499 lines
name: MCG
description: Multi Cloud Gateway / NooBaa related issues
name: Squad/Magenta
name: Squad/Red
name: lgtm
Created: 2025-05-06T14:14:46Z | Merged: 2025-05-16T08:29:54Z



Diff:
diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py
index 79f6cac8cef..991490ad096 100644
--- a/ocs_ci/ocs/constants.py
+++ b/ocs_ci/ocs/constants.py
@@ -709,6 +709,11 @@
 
 # NooBaa DB CNPG
 NB_DB_PRIMARY_POD_LABEL = "cnpg.io/instanceRole=primary"
+CNPG_POD_ROLE_INSTANCE_LABEL = "cnpg.io/podRole=instance"
+CNPG_CLUSTER_KIND = "cluster"
+NB_DB_CNPG_CLUSTER_NAME = "noobaa-db-pg-cluster"
+NB_DB_CNPG_HEALTHY_STATUS = "Cluster in healthy state"
+NB_DB_CNPG_APP_SECRET = "noobaa-db-pg-cluster-app"
 
 # Auth Yaml
 OCSCI_DATA_BUCKET = "ocs-ci-data"
diff --git a/tests/cross_functional/conftest.py b/tests/cross_functional/conftest.py
index ef83a66fb21..aceea817827 100644
--- a/tests/cross_functional/conftest.py
+++ b/tests/cross_functional/conftest.py
@@ -6,7 +6,6 @@
 from concurrent.futures import ThreadPoolExecutor
 from threading import Event
 
-from ocs_ci.utility import version
 from ocs_ci.utility.retry import retry
 from ocs_ci.framework import config
 from ocs_ci.helpers.e2e_helpers import (
@@ -37,12 +36,13 @@
 from ocs_ci.ocs.resources import pod, pvc
 from ocs_ci.ocs.resources.objectbucket import OBC
 from ocs_ci.ocs.resources.ocs import OCS
+from ocs_ci.ocs.resources.deployment import Deployment
 from ocs_ci.ocs.resources.pod import (
     Pod,
+    get_noobaa_pods,
+    get_pod_logs,
     get_pods_having_label,
 )
-from ocs_ci.ocs.resources.deployment import Deployment
-from ocs_ci.ocs.resources.pod import get_noobaa_pods, get_pod_logs
 from ocs_ci.ocs.resources.pvc import get_pvc_objs
 from ocs_ci.ocs.exceptions import CommandFailed
 from ocs_ci.helpers.helpers import (
@@ -53,7 +53,7 @@
 )
 from ocs_ci.ocs.ocp import OCP
 from ocs_ci.utility.kms import is_kms_enabled
-from ocs_ci.utility.utils import clone_notify
+from ocs_ci.utility.utils import clone_notify, exec_nb_db_query, get_primary_nb_db_pod
 
 
 logger = logging.getLogger(__name__)
@@ -146,23 +146,14 @@ def factory(
             kind="secret", namespace=config.ENV_DATA["cluster_namespace"]
         )
         secrets = [
+            "noobaa-root-master-key-volume",
+            "noobaa-root-master-key-backend",
             "noobaa-admin",
             "noobaa-operator",
-            "noobaa-db",
             "noobaa-server",
             "noobaa-endpoints",
         ]
 
-        if (
-            version.get_semantic_ocs_version_from_config() >= version.VERSION_4_14
-            and not is_kms_enabled()
-        ):
-            secrets.extend(
-                ["noobaa-root-master-key-backend", "noobaa-root-master-key-volume"]
-            )
-        elif not is_kms_enabled():
-            secrets.append("noobaa-root-master-key")
-
         secrets_yaml = [
             ocp_secret_obj.get(resource_name=f"{secret}") for secret in secrets
         ]
@@ -170,21 +161,23 @@ def factory(
         logger.info("Backed up secrets as secret objects!")
 
         # Backup the PostgreSQL database and save it to a local folder
-        noobaa_db_pod = Pod(
-            **get_pods_having_label(
-                label=constants.NOOBAA_DB_LABEL_47_AND_ABOVE,
-                namespace=config.ENV_DATA["cluster_namespace"],
-            )[0]
-        )
+        noobaa_db_pod = get_primary_nb_db_pod()
         noobaa_db_pod.exec_cmd_on_pod(
-            command="pg_dump nbcore -f /tmp/test.db -F custom"
+            command="pg_dump nbcore -F custom -f /dev/shm/test.db",
         )
         OCP(namespace=config.ENV_DATA["cluster_namespace"]).exec_oc_cmd(
-            command=f"cp --retries=-1 {noobaa_db_pod.name}:/tmp/test.db ./mcg.bck",
+            command=f"cp --retries=-1 {noobaa_db_pod.name}:/dev/shm/test.db ./mcg.bck",
             out_yaml_format=False,
         )
         logger.info("Backed up PostgreSQL and stored it in local folder!")
 
+        # Backup the noobaa-db-pg-cluster resource
+        cnpg_cluster_yaml = OCP(
+            kind=constants.CNPG_CLUSTER_KIND,
+            namespace=config.ENV_DATA["cluster_namespace"],
+        ).get(resource_name=constants.NB_DB_CNPG_CLUSTER_NAME)
+        original_db_replica_count = cnpg_cluster_yaml["spec"]["instances"]
+
         # For testing, write new data to show a little data loss between backup and restore
         testloss_bucket = bucket_factory()[0]
         write_random_test_objects_to_bucket(
@@ -216,17 +209,59 @@ def factory(
 
         # Login to the NooBaa DB pod and cleanup potential database clients to nbcore
         query = "SELECT pg_terminate_backend (pid) FROM pg_stat_activity WHERE datname = 'nbcore';"
-        noobaa_db_pod.exec_cmd_on_pod(
-            command=f'psql -h 127.0.0.1 -p 5432 -U postgres -c "{query}"'
-        )
-        logger.info("Cleaned up potential database clients to nbcore!")
+        try:
+            exec_nb_db_query(query)
+        except CommandFailed as ex:
+            if "terminating connection due to administrator command" not in str(ex):
+                raise ex
+            logger.info("Cleaned up potential database clients to nbcore!")
+
+        # Delete the existing cnpg cluster
+        OCP(kind=constants.CNPG_CLUSTER_KIND).delete(
+            resource_name=constants.NB_DB_CNPG_CLUSTER_NAME
+        )
+
+        # Ensure the the cnpg cluster yaml uses the correct bootstrap object
+        cnpg_cluster_yaml["bootstrap"] = {
+            "initdb": {
+                "database": "nbcore",
+                "encoding": "UTF8",
+                "localeCType": "C",
+                "localeCollate": "C",
+                "owner": "noobaa",
+            }
+        }
+        cnpg_cluster_obj = OCS(**cnpg_cluster_yaml)
+        cnpg_cluster_obj.create()
 
-        # Restore DB from a local folder
-        OCP(namespace=config.ENV_DATA["cluster_namespace"]).exec_oc_cmd(
-            command=f"cp ./mcg.bck {noobaa_db_pod.name}:/tmp/test.db"
+        # Wait for the cluster status to be in a healthy state
+        selector = (
+            f"{constants.NOOBAA_DB_LABEL_419_AND_ABOVE},"
+            f"{constants.CNPG_POD_ROLE_INSTANCE_LABEL}"
+        )
+        OCP(kind=constants.POD).wait_for_resource(
+            condition=constants.STATUS_RUNNING,
+            selector=selector,
+            resource_count=original_db_replica_count,
+            timeout=600,
+            sleep=5,
         )
-        noobaa_db_pod.exec_cmd_on_pod(command="pg_restore -d nbcore /tmp/test.db -c")
-        logger.info("Restored DB from the local folder!")
+
+        # Restore DB from a local folder to the primary instance
+        for pod_info in get_pods_having_label(label=constants.NOOBAA_CNPG_POD_LABEL):
+            noobaa_db_pod = Pod(**pod_info)
+            noobaa_db_pod = get_primary_nb_db_pod()
+            OCP(namespace=config.ENV_DATA["cluster_namespace"]).exec_oc_cmd(
+                command=f"cp --retries=-1 ./mcg.bck {noobaa_db_pod.name}:/dev/shm/test.db",
+                out_yaml_format=False,
+            )
+            cmd = (
+                'bash -c "pg_restore --no-owner -n public '
+                "--role=noobaa -d nbcore "
+                '--verbose < /dev/shm/test.db"'
+            )
+            noobaa_db_pod.exec_cmd_on_pod(command=cmd)
+            logger.info(f"Restored {noobaa_db_pod.name} from the local folder!")
 
         # Delete secrets and restore them from a local folder.
         # Please note that verify that there are no errors before you proceed to the next steps.
@@ -288,11 +323,12 @@ def finalizer():
             logger.info("Removed the local copy of mcg.bck")
 
         # create the secrets if they're deleted
-        for secret in secrets_obj:
-            if secret.is_deleted:
-                secret.create()
-            else:
-                logger.info(f"{secret.name} is not deleted!")
+        if secrets_obj:
+            for secret in secrets_obj:
+                if secret.is_deleted:
+                    secret.create()
+                else:
+                    logger.info(f"{secret.name} is not deleted!")
 
         # restore MCG reconcilation if not restored already
         if (
diff --git a/tests/cross_functional/kcs/test_noobaa_db_backup_and_recovery.py b/tests/cross_functional/kcs/test_noobaa_db_backup_and_recovery.py
index df866dab22c..a0935b0d3d4 100644
--- a/tests/cross_functional/kcs/test_noobaa_db_backup_and_recovery.py
+++ b/tests/cross_functional/kcs/test_noobaa_db_backup_and_recovery.py
@@ -90,7 +90,7 @@ def warps3(self, request):
         warps3.create_resource_warp(replicas=4, multi_client=True)
 
         def teardown():
-            warps3.cleanup()
+            warps3.cleanup(multi_client=True)
 
         request.addfinalizer(teardown)
         return warps3


Reviews:
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           shirshfe-419ibm09
Cluster Configuration:  conf/deployment/ibmcloud/ipi_3az_rhcos_3m_3w.yaml
PR Test Suite:          
PR Test Path:           tests/cross_functional/kcs/test_noobaa_db_backup_and_recovery.py::TestNoobaaBackupAndRecovery::test_noobaa_db_backup_recovery_locally
Additional Test Params: 
OCP VERSION:            4.19
OCS VERSION:            4.19
tested against branch:  master


 [Job UNSTABLE](https://url.corp.redhat.com/b831917) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-14T23:05:31Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           shirshfe-419ibm10
Cluster Configuration:  conf/deployment/ibmcloud/ipi_3az_rhcos_3m_3w.yaml
PR Test Suite:          
PR Test Path:           tests/cross_functional/kcs/test_noobaa_db_backup_and_recovery.py::TestNoobaaBackupAndRecovery::test_noobaa_db_backup_recovery_locally
Additional Test Params: 
OCP VERSION:            4.19
OCS VERSION:            4.19
tested against branch:  master


 [Job PASSED](https://url.corp.redhat.com/b417bcd).
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-15T00:06:00Z
user: login: udaysk23
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T10:10:36Z
user: login: mashetty330
body: Thanks @sagihirshfeld for addressing this quickly. 
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T10:38:39Z
user: login: sagihirshfeld
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T11:45:56Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           shirshfe-419ibm10
Cluster Configuration:  conf/deployment/ibmcloud/ipi_3az_rhcos_3m_3w.yaml
PR Test Suite:          
PR Test Path:           tests/cross_functional/kcs/test_noobaa_db_backup_and_recovery.py::TestNoobaaBackupAndRecovery::test_noobaa_db_backup_recovery_locally
Additional Test Params: 
OCP VERSION:            4.19
OCS VERSION:            4.19
tested against branch:  master


 [Job UNSTABLE](https://url.corp.redhat.com/8223352) (some or all tests failed).
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-15T13:16:01Z
user: login: ocs-ci
body: PR validation on existing cluster 


Cluster Name:           shirshfe-419ibm10
Cluster Configuration:  conf/deployment/ibmcloud/ipi_3az_rhcos_3m_3w.yaml
PR Test Suite:          
PR Test Path:           tests/cross_functional/kcs/test_noobaa_db_backup_and_recovery.py::TestNoobaaBackupAndRecovery::test_noobaa_db_backup_recovery_locally
Additional Test Params: 
OCP VERSION:            4.19
OCS VERSION:            4.19
tested against branch:  master


 [Job PASSED](https://url.corp.redhat.com/02b2345).
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-15T14:45:48Z
user: login: sagihirshfeld
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T14:51:51Z
user: login: mashetty330
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T18:42:17Z
user: login: PrasadDesala
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-16T06:01:05Z

Review comments:
diff_hunk: @@ -215,18 +209,72 @@ def factory(
         )
 
         # Login to the NooBaa DB pod and cleanup potential database clients to nbcore
-        query = "SELECT pg_terminate_backend (pid) FROM pg_stat_activity WHERE datname = 'nbcore';"
-        noobaa_db_pod.exec_cmd_on_pod(
-            command=f'psql -h 127.0.0.1 -p 5432 -U postgres -c "{query}"'
+        nb_db_pg_cluster_app_secret = OCP(
+            kind=constants.SECRET,
+            resource_name=constants.NB_DB_CNPG_APP_SECRET,
         )
-        logger.info("Cleaned up potential database clients to nbcore!")
+        pg_password = b64decode(
path: tests/cross_functional/conftest.py
user: login: mashetty330
body: @sagihirshfeld what is this password? how different is this from running normal db query?
created_at: 2025-05-15T10:37:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -215,18 +209,72 @@ def factory(
         )
 
         # Login to the NooBaa DB pod and cleanup potential database clients to nbcore
-        query = "SELECT pg_terminate_backend (pid) FROM pg_stat_activity WHERE datname = 'nbcore';"
-        noobaa_db_pod.exec_cmd_on_pod(
-            command=f'psql -h 127.0.0.1 -p 5432 -U postgres -c "{query}"'
+        nb_db_pg_cluster_app_secret = OCP(
+            kind=constants.SECRET,
+            resource_name=constants.NB_DB_CNPG_APP_SECRET,
         )
-        logger.info("Cleaned up potential database clients to nbcore!")
+        pg_password = b64decode(
path: tests/cross_functional/conftest.py
user: login: sagihirshfeld
body: I got an authorization error for this specific command without it. This command wasn't a part of the old or new procedures, so I tried to leave it as-is in case it was a specific test requirement I wasn't aware of:
```
"SELECT pg_terminate_backend (pid) FROM pg_stat_activity WHERE datname = 'nbcore';"
```
created_at: 2025-05-15T11:45:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -215,18 +209,72 @@ def factory(
         )
 
         # Login to the NooBaa DB pod and cleanup potential database clients to nbcore
-        query = "SELECT pg_terminate_backend (pid) FROM pg_stat_activity WHERE datname = 'nbcore';"
-        noobaa_db_pod.exec_cmd_on_pod(
-            command=f'psql -h 127.0.0.1 -p 5432 -U postgres -c "{query}"'
+        nb_db_pg_cluster_app_secret = OCP(
+            kind=constants.SECRET,
+            resource_name=constants.NB_DB_CNPG_APP_SECRET,
         )
-        logger.info("Cleaned up potential database clients to nbcore!")
+        pg_password = b64decode(
path: tests/cross_functional/conftest.py
user: login: sagihirshfeld
body: Apparently we didn't need this password to run this query - I just switched to using `exec_nb_db_query` in https://github.com/red-hat-storage/ocs-ci/pull/12084/commits/ffc97edfc0d891f4228f18d5cd7d33cd1dc1141e and it still works, as can be seen in the below passing verification job
created_at: 2025-05-15T14:51:50Z
author_association: CONTRIBUTOR

PR comments:
user: login: openshift-ci[bot]
created_at: 2025-05-16T06:01:10Z
body: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/red-hat-storage/ocs-ci/pull/12084#pullrequestreview-2844681016" title="Approved">mashetty330</a>*, *<a href="https://github.com/red-hat-storage/ocs-ci/pull/12084#pullrequestreview-2845580834" title="Approved">PrasadDesala</a>*, *<a href="https://github.com/red-hat-storage/ocs-ci/pull/12084#" title="Author self-approved">sagihirshfeld</a>*


The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=red-hat-storage%2Focs-ci).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/red-hat-storage/ocs-ci/blob/master/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: NONE