Repository: rucio/rucio

Issue:
Protection in rucio-admin rse delete
We need a protection for deleting a RSE. In particular, it should fail if some rules, collection replicas exist.

_Migrated from JIRA [RUCIO-2103](https://its.cern.ch/jira/browse/RUCIO-2103)_


Discussion:
I am not sure how to efficiently check if the rse is empty. If I understood correctly, I could either check for rules like:
```
[rule for rule in list_rules() for rse in parse_expression(rule['rse_expression']) if rse['rse'] == rse] == 0
```
But there I would have to iterate through each rule.

Or to check if there are file or collection replicas like:
```
session.query(models.CollectionReplicas).filter_by(rse_id=rse_id).count() == 0                                                                                                                                                                                                     
session.query(models.RSEFileAssociation).filter_by(rse_id=rse_id).count() == 0
```
Hi,

The best way would be to check against the replicas table directly, thus:

`session.query(models. RSEFileAssociation).filter_by(rse_id=rse_id).count() == 0`

This query goes against the Primary Key directly, so it should be rather fast, especially if the rse is empty.
Okay thanks, was not sure because of the big size of the RSEFileAssociation table.
For this it should be fine, as deleting an RSE is a really rare thing to do. 

Resolving patch:
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -48,7 +48,7 @@
 
 import rucio.core.account_counter
 
-from rucio.core.rse_counter import add_counter
+from rucio.core.rse_counter import add_counter, get_counter
 
 from rucio.common import exception, utils
 from rucio.common.config import get_lfn2pfn_algorithm_default
@@ -164,8 +164,11 @@ def del_rse(rse, session=None):
     :param session: The database session in use.
     """
 
+    old_rse = None
     try:
         old_rse = session.query(models.RSE).filter_by(rse=rse).one()
+        if not rse_is_empty(rse=rse, session=session):
+            raise exception.RSEOperationNotSupported('RSE \'%s\' is not empty' % rse)
     except sqlalchemy.orm.exc.NoResultFound:
         raise exception.RSENotFound('RSE \'%s\' cannot be found' % rse)
     old_rse.delete(session=session)
@@ -175,6 +178,19 @@ def del_rse(rse, session=None):
         pass
 
 
+@read_session
+def rse_is_empty(rse, session=None):
+    """
+    Check if a RSE is empty.
+
+    :param rse: the rse name.
+    :param session: the database session in use.
+    """
+
+    rse_id = get_rse(rse, session=session)['id']
+    return get_counter(rse_id, session=session)['bytes'] == 0
+
+
 @read_session
 def get_rse(rse, rse_id=None, session=None):
     """
diff --git a/lib/rucio/web/rest/flaskapi/v1/rse.py b/lib/rucio/web/rest/flaskapi/v1/rse.py
--- a/lib/rucio/web/rest/flaskapi/v1/rse.py
+++ b/lib/rucio/web/rest/flaskapi/v1/rse.py
@@ -42,7 +42,7 @@
                                     RSEOperationNotSupported, RSEProtocolNotSupported,
                                     InvalidObject, RSEProtocolDomainNotSupported,
                                     RSEProtocolPriorityError, InvalidRSEExpression,
-                                    RSEAttributeNotFound)
+                                    RSEAttributeNotFound, CounterNotFound)
 from rucio.common.utils import generate_http_error_flask, render_json, APIEncoder
 from rucio.web.rest.flaskapi.v1.common import before_request, after_request
 from rucio.rse import rsemanager
@@ -235,6 +235,10 @@ def delete(self, rse):
             return generate_http_error_flask(404, 'RSENotFound', error.args[0])
         except AccessDenied as error:
             return generate_http_error_flask(401, 'AccessDenied', error.args[0])
+        except RSEOperationNotSupported as error:
+            return generate_http_error_flask(404, 'RSEOperationNotsupported', error.args[0])
+        except CounterNotFound as error:
+            return generate_http_error_flask(404, 'CounterNotFound', error.args[0])
 
         return "OK", 200
 
diff --git a/lib/rucio/web/rest/webpy/v1/rse.py b/lib/rucio/web/rest/webpy/v1/rse.py
--- a/lib/rucio/web/rest/webpy/v1/rse.py
+++ b/lib/rucio/web/rest/webpy/v1/rse.py
@@ -46,7 +46,7 @@
                                     RSEOperationNotSupported, RSEProtocolNotSupported,
                                     InvalidObject, RSEProtocolDomainNotSupported,
                                     RSEProtocolPriorityError, InvalidRSEExpression,
-                                    RSEAttributeNotFound)
+                                    RSEAttributeNotFound, CounterNotFound)
 from rucio.common.utils import generate_http_error, render_json, APIEncoder
 from rucio.web.rest.common import rucio_loadhook, RucioController
 from rucio.rse import rsemanager
@@ -238,8 +238,12 @@ def DELETE(self, rse):
             del_rse(rse=rse, issuer=ctx.env.get('issuer'))
         except RSENotFound as error:
             raise generate_http_error(404, 'RSENotFound', error.args[0])
+        except RSEOperationNotSupported as error:
+            raise generate_http_error(404, 'RSEOperationNotSupported', error.args[0])
         except AccessDenied as error:
             raise generate_http_error(401, 'AccessDenied', error.args[0])
+        except CounterNotFound as error:
+            raise generate_http_error(404, 'CounterNotFound', error.args[0])
 
         raise OK()
 


Test patch:
diff --git a/lib/rucio/tests/test_archive.py b/lib/rucio/tests/test_archive.py
--- a/lib/rucio/tests/test_archive.py
+++ b/lib/rucio/tests/test_archive.py
@@ -17,6 +17,7 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2017
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 
 from nose.tools import assert_equal, assert_in, assert_not_in
 
@@ -24,7 +25,7 @@
 from rucio.client.replicaclient import ReplicaClient
 from rucio.common.utils import generate_uuid
 from rucio.core.replica import add_replicas
-from rucio.core.rse import add_rse, del_rse, add_protocol
+from rucio.core.rse import add_rse, add_protocol
 from rucio.tests.common import rse_name_generator
 
 
@@ -107,8 +108,6 @@ def test_list_archive_contents_transparently(self):
         for r in res:
             assert_in('weighted.storage.cube.zip?xrdcl.unzip=norep-', r.keys()[0])
 
-        del_rse(rse)
-
     def test_list_archive_contents_at_rse(self):
         """ ARCHIVE (CORE): Transparent archive listing at RSE """
 
@@ -159,6 +158,3 @@ def test_list_archive_contents_at_rse(self):
         res = self.rc.list_replicas(dids=[{'scope': scope, 'name': f['name']} for f in archived_file], metalink=True, rse_expression=rse2, resolve_archives=True)
         assert_in('BLACKMESA', res)
         assert_not_in('APERTURE', res)
-
-        del_rse(rse1)
-        del_rse(rse2)
diff --git a/lib/rucio/tests/test_dataset_replicas.py b/lib/rucio/tests/test_dataset_replicas.py
--- a/lib/rucio/tests/test_dataset_replicas.py
+++ b/lib/rucio/tests/test_dataset_replicas.py
@@ -16,11 +16,12 @@
 # - Vincent Garonne <vgaronne@gmail.com>, 2015
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2015
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 
 from nose.tools import assert_equal
 
 from rucio.core.replica import list_datasets_per_rse
-from rucio.core.rse import add_rse, del_rse, add_protocol
+from rucio.core.rse import add_rse, add_protocol
 
 from rucio.client.didclient import DIDClient
 from rucio.client.replicaclient import ReplicaClient
@@ -122,5 +123,3 @@ def test_list_dataset_replicas_archive(self):
         assert_equal(res[0]['state'], 'AVAILABLE')
         assert_equal(res[1]['state'], 'AVAILABLE')
         assert_equal(res[2]['state'], 'AVAILABLE')
-
-        del_rse(rse)
diff --git a/lib/rucio/tests/test_rse.py b/lib/rucio/tests/test_rse.py
--- a/lib/rucio/tests/test_rse.py
+++ b/lib/rucio/tests/test_rse.py
@@ -32,16 +32,17 @@
 from nose.tools import raises, assert_equal, assert_true, assert_in, assert_raises
 from paste.fixture import TestApp
 
+from rucio.db.sqla import session, models
 from rucio.db.sqla.constants import RSEType
 from rucio.client.rseclient import RSEClient
 from rucio.client.replicaclient import ReplicaClient
 from rucio.common.exception import (Duplicate, RSENotFound, RSEProtocolNotSupported,
                                     InvalidObject, RSEProtocolDomainNotSupported, RSEProtocolPriorityError,
-                                    ResourceTemporaryUnavailable, RSEAttributeNotFound)
+                                    ResourceTemporaryUnavailable, RSEAttributeNotFound, RSEOperationNotSupported)
 from rucio.common.utils import generate_uuid
 from rucio.core.rse import (add_rse, get_rse_id, del_rse, list_rses, rse_exists, add_rse_attribute, list_rse_attributes,
                             set_rse_transfer_limits, get_rse_transfer_limits, delete_rse_transfer_limits,
-                            get_rse_protocols, del_rse_attribute, get_rse_attribute, get_rse)
+                            get_rse_protocols, del_rse_attribute, get_rse_attribute, get_rse, rse_is_empty)
 from rucio.rse import rsemanager as mgr
 from rucio.tests.common import rse_name_generator
 from rucio.web.rest.rse import APP as rse_app
@@ -162,6 +163,37 @@ def test_delete_rse_attribute(self):
         with assert_raises(RSEAttributeNotFound):
             del_rse_attribute(rse=rse_name, key=rse_name)
 
+    def test_delete_rse(self):
+        """ RSE (CORE): Test deletion of RSE """
+        # Deletion of not empty RSE
+        rse_name = rse_name_generator()
+        add_rse(rse_name)
+        rse_id = get_rse_id(rse_name)
+        db_session = session.get_session()
+        rse_usage = db_session.query(models.RSEUsage).filter_by(rse_id=rse_id, source='rucio').one()
+        rse_usage.used = 1
+        db_session.commit()
+        with assert_raises(RSEOperationNotSupported):
+            del_rse(rse_name)
+
+        # Deletion of not found RSE:
+        rse_name = rse_name_generator()
+        with assert_raises(RSENotFound):
+            del_rse(rse_name)
+
+    def test_empty_rse(self):
+        """ RSE (CORE): Test if RSE is empty """
+        rse_name = rse_name_generator()
+        add_rse(rse_name)
+        rse_id = get_rse_id(rse_name)
+        assert_equal(rse_is_empty(rse_name), True)
+
+        db_session = session.get_session()
+        rse_usage = db_session.query(models.RSEUsage).filter_by(rse_id=rse_id, source='rucio').one()
+        rse_usage.used = 1
+        db_session.commit()
+        assert_equal(rse_is_empty(rse_name), False)
+
 
 class TestRSE(object):
 
@@ -273,13 +305,13 @@ def test_get_rse_account_usage(self):
 
     def test_delete_rse_attribute(self):
         """ RSE (REST): Test the deletion of a RSE attribute """
+        rse_name = rse_name_generator()
+        add_rse(rse_name)
         mw = []
         headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': 'ddmlab', 'X-Rucio-Password': 'secret'}
         r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
         token = str(r1.header('X-Rucio-Auth-Token'))
 
-        rse_name = rse_name_generator()
-        add_rse(rse_name)
         headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}
         r2 = TestApp(rse_app.wsgifunc(*mw)).delete('/{0}/attr/{0}'.format(rse_name), headers=headers2, expect_errors=True)
         assert_equal(r2.status, 200)
@@ -287,6 +319,42 @@ def test_delete_rse_attribute(self):
         r2 = TestApp(rse_app.wsgifunc(*mw)).delete('/{0}/attr/{0}'.format(rse_name), headers=headers2, expect_errors=True)
         assert_equal(r2.status, 404)
 
+    def test_delete_rse(self):
+        """ RSE (REST): Test the deletion of RSE """
+        mw = []
+        headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': 'ddmlab', 'X-Rucio-Password': 'secret'}
+        r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
+        token = str(r1.header('X-Rucio-Auth-Token'))
+
+        # Normal deletion
+        rse_name = rse_name_generator()
+        add_rse(rse_name)
+        headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}
+        r2 = TestApp(rse_app.wsgifunc(*mw)).delete('/{0}'.format(rse_name), headers=headers2, expect_errors=True)
+        assert_equal(r2.status, 200)
+
+        # Second deletion
+        headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}
+        r2 = TestApp(rse_app.wsgifunc(*mw)).delete('/{0}'.format(rse_name), headers=headers2, expect_errors=True)
+        assert_equal(r2.status, 404)
+
+        # Deletion of not found RSE
+        rse_name = rse_name_generator()
+        headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}
+        r2 = TestApp(rse_app.wsgifunc(*mw)).delete('/{0}'.format(rse_name), headers=headers2, expect_errors=True)
+        assert_equal(r2.status, 404)
+
+        # Deletion of not empty RSE
+        rse_name = rse_name_generator()
+        add_rse(rse_name)
+        rse_id = get_rse_id(rse_name)
+        db_session = session.get_session()
+        rse_usage = db_session.query(models.RSEUsage).filter_by(rse_id=rse_id, source='rucio').one()
+        rse_usage.used = 1
+        db_session.commit()
+        headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}
+        r2 = TestApp(rse_app.wsgifunc(*mw)).delete('/{0}'.format(rse_name), headers=headers2, expect_errors=True)
+
 
 class TestRSEClient(object):
 
@@ -1417,7 +1485,6 @@ def test_get_rse_protocols_includes_verify_checksum(self):
 
         assert_in('verify_checksum', info)
         assert_equal(info['verify_checksum'], True)
-
         del_rse(rse)
 
     def test_delete_rse_attribute(self):
@@ -1429,3 +1496,23 @@ def test_delete_rse_attribute(self):
 
         with assert_raises(RSEAttributeNotFound):
             self.client.delete_rse_attribute(rse=rse_name, key=rse_name)
+
+    def test_delete_rse(self):
+        """ RSE (CLIENTS): delete RSE """
+        # Deletion of not empty RSE
+        rse_name = rse_name_generator()
+        add_rse(rse_name)
+        rse_id = get_rse_id(rse_name)
+        db_session = session.get_session()
+        rse_usage = db_session.query(models.RSEUsage).filter_by(rse_id=rse_id, source='rucio').one()
+        rse_usage.used = 1
+        db_session.commit()
+        db_session = session.get_session()
+        print(db_session.query(models.RSEUsage).filter_by(rse_id=rse_id).one())
+        with assert_raises(RSEOperationNotSupported):
+            self.client.delete_rse(rse=rse_name)
+
+        # Deletion of not found RSE:
+        rse_name = rse_name_generator()
+        with assert_raises(RSENotFound):
+            self.client.delete_rse(rse=rse_name)