Repository: rucio/rucio

Issue:
RSE's add_protocol is leaking database exceptions on the API
Motivation
----------
Tests are failing after removing some of the broader pytest xfail decorators and broad exception clauses. In particular the server-side of `rseclient.add_protocol` is propagating the correct error for the Oracle database, but the tests are not running on MySQL and Postgres, because their `IntegrityError`s aren't properly handled. This affects the following two tests:

* lib/rucio/tests/test_rse.py::TestRSEClient::test_add_protocol_missing_values
* lib/rucio/tests/test_rse.py::TestRSEClient::test_update_protocols_invalid_value


Modification
---
Catch the database errors on the server and throw `rucio.common.exception.InvalidObject` instead.
RSE's add_protocol is leaking database exceptions on the API
Motivation
----------
Tests are failing after removing some of the broader pytest xfail decorators and broad exception clauses. In particular the server-side of `rseclient.add_protocol` is propagating the correct error for the Oracle database, but the tests are not running on MySQL and Postgres, because their `IntegrityError`s aren't properly handled. This affects the following two tests:

* lib/rucio/tests/test_rse.py::TestRSEClient::test_add_protocol_missing_values
* lib/rucio/tests/test_rse.py::TestRSEClient::test_update_protocols_invalid_value


Modification
---
Catch the database errors on the server and throw `rucio.common.exception.InvalidObject` instead.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -930,17 +930,21 @@ def add_protocol(rse_id, parameter, session=None):
         new_protocol.save(session=session)
     except (IntegrityError, FlushError, OperationalError) as error:
         if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \
-           or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \
-           or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
-           or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\
-           or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0])\
-           or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]):
+                or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \
+                or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
+                or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \
+                or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \
+                or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]):
             raise exception.Duplicate('Protocol \'%s\' on port %s already registered for  \'%s\' with hostname \'%s\'.' % (parameter['scheme'], parameter['port'], rse, parameter['hostname']))
         elif 'may not be NULL' in error.args[0] \
-             or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \
-             or match('.*OperationalError.*cannot be null.*', error.args[0]):
+                or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \
+                or match('.*IntegrityError.*Column.*cannot be null.*', error.args[0]) \
+                or match('.*IntegrityError.*null value in column.*violates not-null constraint.*', error.args[0]) \
+                or match('.*NotNullViolation.*null value in column.*violates not-null constraint.*', error.args[0]) \
+                or match('.*OperationalError.*cannot be null.*', error.args[0]):
             raise exception.InvalidObject('Missing values!')
-        raise error
+
+        raise exception.RucioException(error.args)
     return new_protocol
 
 


Test patch:
diff --git a/lib/rucio/tests/test_rse.py b/lib/rucio/tests/test_rse.py
--- a/lib/rucio/tests/test_rse.py
+++ b/lib/rucio/tests/test_rse.py
@@ -34,7 +34,6 @@
 
 from __future__ import print_function
 
-import os
 import unittest
 from json import dumps
 
@@ -46,12 +45,14 @@
 from rucio.common import exception
 from rucio.common.config import config_get, config_get_bool
 from rucio.common.exception import (Duplicate, RSENotFound, RSEProtocolNotSupported,
-                                    InvalidObject, ResourceTemporaryUnavailable, RSEAttributeNotFound, RSEOperationNotSupported)
+                                    InvalidObject, ResourceTemporaryUnavailable,
+                                    RSEAttributeNotFound, RSEOperationNotSupported)
 from rucio.common.utils import generate_uuid
-from rucio.core.rse import (add_rse, get_rse_id, del_rse, restore_rse, list_rses, rse_exists, add_rse_attribute,
-                            list_rse_attributes,
-                            set_rse_transfer_limits, get_rse_transfer_limits, delete_rse_transfer_limits,
-                            get_rse_protocols, del_rse_attribute, get_rse_attribute, get_rse, rse_is_empty)
+from rucio.core.rse import (add_rse, get_rse_id, del_rse, restore_rse, list_rses,
+                            rse_exists, add_rse_attribute, list_rse_attributes,
+                            set_rse_transfer_limits, get_rse_transfer_limits,
+                            delete_rse_transfer_limits, get_rse_protocols,
+                            del_rse_attribute, get_rse_attribute, get_rse, rse_is_empty)
 from rucio.db.sqla import session, models
 from rucio.db.sqla.constants import RSEType
 from rucio.rse import rsemanager as mgr
@@ -606,16 +607,6 @@ def test_add_protocol_missing_values(self):
         try:
             with pytest.raises(exception.InvalidObject):
                 self.client.add_protocol(protocol_rse, attributes)
-        except exception.DatabaseException:
-            if 'RDBMS' in os.environ:
-                if os.environ['RDBMS'].startswith('mysql'):
-                    pytest.xfail("Uncaught MySQL exception on server-side, bug https://github.com/rucio/rucio/issues/3921")
-                elif os.environ['RDBMS'].startswith('postgres'):
-                    pytest.xfail("Uncaught Postgres exception on server-side, bug https://github.com/rucio/rucio/issues/3921")
-                else:
-                    raise
-            else:
-                raise
         finally:
             self.client.delete_rse(protocol_rse)
 
@@ -1361,16 +1352,6 @@ def test_update_protocols_invalid_value(self):
 
             with pytest.raises(exception.RSEProtocolNotSupported):
                 self.client.update_protocols(protocol_rse, scheme=attributes['scheme'], hostname=attributes['hostname'], port=attributes['port'], data={'impl': None})
-        except exception.DatabaseException:
-            if 'RDBMS' in os.environ:
-                if os.environ['RDBMS'].startswith('mysql'):
-                    pytest.xfail("Uncaught MySQL exception on server-side, bug https://github.com/rucio/rucio/issues/3921")
-                elif os.environ['RDBMS'].startswith('postgres'):
-                    pytest.xfail("Uncaught Postgres exception on server-side, bug https://github.com/rucio/rucio/issues/3921")
-                else:
-                    raise
-            else:
-                raise
         finally:
             self.client.delete_rse(protocol_rse)