Repository: medtagger/MedTagger

Issue:
Do not drop tables in functional tests
## Expected Behavior

Functional tests should clean the tables in Postgres and HBase.

## Actual Behavior

Functional tests drops all the available tables and create them again. This may be (and probably is) time consuming.

## Additional comment

Please compare both implementations before merge.


Discussion:

Resolving patch:
diff --git a/backend/medtagger/clients/hbase_client.py b/backend/medtagger/clients/hbase_client.py
--- a/backend/medtagger/clients/hbase_client.py
+++ b/backend/medtagger/clients/hbase_client.py
@@ -111,6 +111,21 @@ def get(table_name: str, key: str, columns: List[str] = None) -> Mapping:
             table = connection.table(table_name)
             return table.row(hbase_key, columns=columns)
 
+    @staticmethod
+    @retry(stop_max_attempt_number=3, wait_random_min=200, wait_random_max=1000,
+           retry_on_exception=lambda ex: isinstance(ex, (TTransportException, BrokenPipeError)))
+    def delete(table_name: str, key: str, columns: List[str] = None) -> None:
+        """Delete a single row (or values from colums in given row) in HBase table.
+
+        :param table_name: name of a table
+        :param key: key representing a row
+        :param columns: columns which should be cleared
+        """
+        hbase_key = str.encode(key)
+        with HBASE_CONNECTION_POOL.connection() as connection:
+            table = connection.table(table_name)
+            table.delete(hbase_key, columns=columns)
+
     @staticmethod
     @retry(stop_max_attempt_number=3, wait_random_min=200, wait_random_max=1000,
            retry_on_exception=lambda ex: isinstance(ex, (TTransportException, BrokenPipeError)))
diff --git a/backend/scripts/utils.py b/backend/scripts/utils.py
--- a/backend/scripts/utils.py
+++ b/backend/scripts/utils.py
@@ -1,8 +1,13 @@
 """Storage for all utility functions."""
 from starbase import Connection
+from retrying import retry
+import requests
+
 from medtagger.config import AppConfiguration
 
 
+@retry(stop_max_attempt_number=5, wait_random_min=200, wait_random_max=1000,
+       retry_on_exception=lambda ex: isinstance(ex, requests.ConnectionError))
 def get_connection_to_hbase() -> Connection:
     """Fetch configuration data and create HBase connection.
 
@@ -11,7 +16,9 @@ def get_connection_to_hbase() -> Connection:
     configuration = AppConfiguration()
     host = configuration.get('hbase', 'host', fallback='localhost')
     port = configuration.getint('hbase', 'rest_port', fallback=8080)
-    return Connection(host=host, port=port)
+    connection = Connection(host=host, port=port)
+    connection.tables()  # Test if the connection was properly set up
+    return connection
 
 
 def user_agrees(prompt_message: str) -> bool:


Test patch:
diff --git a/backend/tests/functional_tests/conftest.py b/backend/tests/functional_tests/conftest.py
--- a/backend/tests/functional_tests/conftest.py
+++ b/backend/tests/functional_tests/conftest.py
@@ -3,10 +3,8 @@
 from typing import Any
 
 import pytest
-from starbase import Connection
 
-from medtagger.config import AppConfiguration
-from medtagger.database import Base, engine, session
+from medtagger.database import Base, session
 from medtagger.database.fixtures import apply_all_fixtures
 from medtagger.clients.hbase_client import HBaseClient
 
@@ -16,31 +14,22 @@
 @pytest.fixture
 def prepare_environment() -> Any:
     """Prepare environment for testing purpose (setup DBs, fixtures) and clean up after the tests."""
-    logger.info('Getting needed entries')
-    configuration = AppConfiguration()
-    host = configuration.get('hbase', 'host', fallback='localhost')
-    port = configuration.getint('hbase', 'rest_port', fallback=8080)
-
-    logger.info('Preparing databases.')
-    Base.metadata.create_all(engine)
-    hbase_connection = Connection(host=host, port=port)
-    for table_name in HBaseClient.HBASE_SCHEMA:
-        list_of_columns = HBaseClient.HBASE_SCHEMA[table_name]
-        table = hbase_connection.table(table_name)
-        table.create(*list_of_columns)
-
-    logger.info('Applying fixtures')
+    logger.info('Applying fixtures to PostgreSQL.')
     apply_all_fixtures()
 
     # Run the test
     yield
 
-    logger.info('Removing all data.')
+    logger.info('Removing all data from PostgreSQL.')
+    for table in reversed(Base.metadata.sorted_tables):
+        session.execute(table.delete())
+    session.commit()
     session.close_all()
-    Base.metadata.drop_all(engine)
+
+    logger.info('Removing all data from HBase.')
     for table_name in HBaseClient.HBASE_SCHEMA:
-        table = hbase_connection.table(table_name)
-        table.drop()
+        for key in HBaseClient.get_all_keys(table_name):
+            HBaseClient.delete(table_name, key)
 
 
 @pytest.fixture