Repository: red-hat-storage/ocs-ci
Issue: Make sure workload_fio_storageutilization fixtures waits for the storage to be actually reclaimed

When test case(s) in a run use multiple different `workload_storageutilization_*` fixtures, it could happen that while OCS is still deleting data written by one just finished fixture, another storage utilization fixture is already started, thinking that there is a lot of data, but by the time it starts writing,  the space is finaly reclaimed so that the actual global utilization (based on a measurement when the utilization was still high because of the previous fixture) is nowhere near the expected target of the 2nd fixture.



Changed files:
ocs_ci/utility/utils.py, tests/manage/monitoring/conftest.py, tests/manage/monitoring/test_workload_fixture.py

diff --git a/ocs_ci/utility/utils.py b/ocs_ci/utility/utils.py
index 8e262b02..08da76e9 100644
--- a/ocs_ci/utility/utils.py
+++ b/ocs_ci/utility/utils.py
@@ -730,11 +730,14 @@ class TimeoutSampler(object):
             self.last_sample_time = time.time()
             try:
                 yield self.func(*self.func_args, **self.func_kwargs)
-            except Exception:
-                pass
-
+            except Exception as ex:
+                msg = f"Exception raised during iteration: {ex}"
+                logging.error(msg)
             if self.timeout < (time.time() - self.start_time):
                 raise self.timeout_exc_cls(*self.timeout_exc_args)
+            log.info(
+                f"Going to sleep for {self.sleep} seconds"
+                " before next iteration")
             time.sleep(self.sleep)
 
     def wait_for_func_status(self, result):
@@ -759,11 +762,10 @@ class TimeoutSampler(object):
             for res in self:
                 if result == res:
                     return True
-
         except self.timeout_exc_cls:
             log.error(
-                f"({self.func.__name__}) return incorrect status after timeout"
-            )
+                f"({self.func.__name__}) return incorrect status "
+                f"after {self.timeout} second timeout")
             return False
 
 
diff --git a/tests/manage/monitoring/conftest.py b/tests/manage/monitoring/conftest.py
index 045ded1e..e74b63ab 100644
--- a/tests/manage/monitoring/conftest.py
+++ b/tests/manage/monitoring/conftest.py
@@ -17,6 +17,7 @@ from ocs_ci.ocs.resources import pod
 from ocs_ci.ocs.resources.mcg_bucket import S3Bucket
 from ocs_ci.ocs.resources.objectconfigfile import ObjectConfFile
 from ocs_ci.utility.prometheus import PrometheusAPI
+from ocs_ci.utility.utils import TimeoutSampler
 from tests import helpers
 from tests.helpers import create_unique_resource_name
 
@@ -571,10 +572,10 @@ def get_storageutilization_size(target_percentage, ceph_pool_name):
     target = total * target_percentage
     to_utilize = target - ceph_total_stored
     pvc_size = round(to_utilize / 2**30)  # GiB
-    logger.info((
-        f"fixture is going to request {pvc_size} Gi volume "
-        f"to reach {target/2**30} Gi of total cluster utilization, which "
-        f"is {target_percentage*100}% of the total capacity"))
+    logger.info(
+        f"to reach {target/2**30} Gi of total cluster utilization, "
+        f"which is {target_percentage*100}% of the total capacity, "
+        f"utilization job should request and fill {pvc_size} Gi volume")
     return pvc_size
 
 
@@ -725,6 +726,46 @@ def workload_fio_storageutilization(
         # make sure we communicate what is going to happen
         logger.info(f"going to delete {fixture_name} Job")
         fio_job_file.delete()
+        logger.info(
+            f"going to wait a bit to make sure that "
+            f"data written by {fixture_name} Job are really deleted")
+
+        def check_pvc_size():
+            """
+            Check whether data created by the Job were actually deleted.
+            """
+            # By asking again for pvc_size necessary to reach the target
+            # cluster utilization, we can see how much data were already
+            # deleted. Negative or small value of current pvc_size means that
+            # the data were not yet deleted.
+            pvc_size_tmp = get_storageutilization_size(
+                target_percentage, ceph_pool_name)
+            # If no other components were utilizing OCS storage, the space
+            # would be considered reclaimed when current pvc_size reaches
+            # it's original value again. But since this is not the case (eg.
+            # constantly growing monitoring or log data are stored there),
+            # we are ok with just 90% of the original value.
+            result = pvc_size_tmp >= pvc_size * 0.90
+            if result:
+                logger.info("storage space was reclaimed")
+            else:
+                logger.info(
+                    "storage space was not yet fully reclaimed, "
+                    f"current pvc size {pvc_size_tmp} value "
+                    f"should be close to {pvc_size}")
+            return result
+
+        check_timeout = 660  # seconds
+        check_sampler = TimeoutSampler(
+            timeout=check_timeout, sleep=30, func=check_pvc_size)
+        finished_in_time = check_sampler.wait_for_func_status(result=True)
+        if not finished_in_time:
+            error_msg = (
+                "it seems that the storage space was not reclaimed "
+                f"within {check_timeout} seconds, "
+                "this is most likely a product bug or misconfiguration")
+            logger.error(error_msg)
+            raise Exception(error_msg)
 
     return measured_op
 
diff --git a/tests/manage/monitoring/test_workload_fixture.py b/tests/manage/monitoring/test_workload_fixture.py
index b7db54e8..688e8931 100644
--- a/tests/manage/monitoring/test_workload_fixture.py
+++ b/tests/manage/monitoring/test_workload_fixture.py
@@ -133,3 +133,17 @@ def test_workload_cephfs(workload_storageutilization_50p_cephfs):
     Purpose of this test is to make another workload fixture executed as well.
     """
     logger.info(workload_storageutilization_50p_cephfs)
+
+
+@pytest.mark.libtest
+def test_workload_rbd_cephfs(
+    workload_storageutilization_50p_rbd,
+    workload_storageutilization_50p_cephfs
+):
+    """
+    When this test case is executed as the only test case in pytest test run,
+    it can be used to reproduce issue with workload_fio_storageutilization
+    fixtures, see https://github.com/red-hat-storage/ocs-ci/issues/1327
+    """
+    logger.info(workload_storageutilization_50p_rbd)
+    logger.info(workload_storageutilization_50p_cephfs)