Repository: rucio/rucio

Issue:
Importer fails to import RSEs that were deleted
Motivation
----------
I was experimenting with the Rucio-CRIC integration and while doing so, I was disabling and re enabling RSEs.
What I noticed is that when the importer tries to configure an RSE that already exists on the database but is marked as 'Deleted', it will throw an error: 
`Details: RSE with id 'f9c7592cdc6344eabce00e8ea4d2641a' cannot be found`

After further investigation I found out that when the importer sees an RSE on the json it will check if it already exists to avoid creating it again. It does so using the `rse_module.get_rse_id` method. This method though returns the id even if the RSE is already marked as deleted.
https://github.com/rucio/rucio/blob/9388a53bf2b0b9b82b0727d897ef0ab818ade60f/lib/rucio/core/importer.py#L37
Further down on the code that the rse_id (of the deleted RSE) is referenced, of course it throws an error.

Modification
------------
Possible  solution: Check if the RSE is marked as 'deleted' and enable it again before trying to modify it.





Discussion:
We have to make a bit of a policy decision here. Significant operations like deleting an RSE or re-creating it we always considered strictly manual operations, since it can have quite severe impacts. @cserf do you have an opinion how we should handle this with CRIC? Keep it manual?
Well, as it is the importer right now it 'deletes' all RSEs that are not part of the JSON.
i'm still in favour of my old proposal that we should have an RSE state, so not only exists/not_exists, but also something like enabled/suspended/decomissioning/removed_from_cric, etc... right now this can all be handled via RSE attributes, but I think it would make sense to have this as a dedicated RSEState.
As discussed today, we should make the "merge" strategy configurable here.
- If not present in CRIC, delete RSE
- If not present in CRIC, keep RSE and make deletion only manual
@hahahannes, @arisfkiaras can one of you have a look on this please.
Ok, I guess you mean soft delete the RSE?
Also, how do we deal with a "new" RSE with the same name in the first case? Should we reactivate and then update it

Resolving patch:
diff --git a/lib/rucio/core/importer.py b/lib/rucio/core/importer.py
--- a/lib/rucio/core/importer.py
+++ b/lib/rucio/core/importer.py
@@ -15,35 +15,48 @@
 # Authors:
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
+# - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2019
 #
 # PY3K COMPATIBLE
 
 from six import string_types
-from rucio.common.exception import RSEOperationNotSupported, RSENotFound
+from rucio.common.exception import RSEOperationNotSupported
 from rucio.core import rse as rse_module, distance as distance_module, account as account_module, identity as identity_module
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import RSEType, AccountType, IdentityType
 from rucio.db.sqla.session import transactional_session
+from rucio.common.config import config_get
 
 
 @transactional_session
-def import_rses(rses, session=None):
+def import_rses(rses, rse_sync_method='edit', attr_sync_method='edit', protocol_sync_method='edit', session=None):
     new_rses = []
     for rse_name in rses:
         rse = rses[rse_name]
         if isinstance(rse.get('rse_type'), string_types):
             rse['rse_type'] = RSEType.from_string(str(rse['rse_type']))
-        try:
+
+        if rse_module.rse_exists(rse_name, include_deleted=False, session=session):
+            # RSE exists and is active
             rse_id = rse_module.get_rse_id(rse=rse_name, session=session)
-        except RSENotFound:
+            rse_module.update_rse(rse_id=rse_id, parameters=rse, session=session)
+        elif rse_module.rse_exists(rse_name, include_deleted=True, session=session):
+            # RSE exists but in deleted state
+            # Should only modify the RSE if importer is configured for edit or hard sync
+            if rse_sync_method in ['edit', 'hard']:
+                rse_id = rse_module.get_rse_id(rse=rse_name, include_deleted=True, session=session)
+                rse_module.restore_rse(rse_id, session=session)
+                rse_module.update_rse(rse_id=rse_id, parameters=rse, session=session)
+            else:
+                # Config is in RSE append only mode, should not modify the disabled RSE
+                continue
+        else:
             rse_id = rse_module.add_rse(rse=rse_name, deterministic=rse.get('deterministic'), volatile=rse.get('volatile'),
                                         city=rse.get('city'), region_code=rse.get('region_code'), country_name=rse.get('country_name'),
                                         staging_area=rse.get('staging_area'), continent=rse.get('continent'), time_zone=rse.get('time_zone'),
                                         ISP=rse.get('ISP'), rse_type=rse.get('rse_type'), latitude=rse.get('latitude'),
                                         longitude=rse.get('longitude'), ASN=rse.get('ASN'), availability=rse.get('availability'),
                                         session=session)
-        else:
-            rse_module.update_rse(rse_id=rse_id, parameters=rse, session=session)
 
         new_rses.append(rse_id)
         # Protocols
@@ -55,6 +68,10 @@ def import_rses(rses, session=None):
             outdated_protocols = [new_protocol for new_protocol in new_protocols if {'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port']} in old_protocols]
             new_protocols = [{'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port']} for protocol in new_protocols]
             to_be_removed_protocols = [old_protocol for old_protocol in old_protocols if old_protocol not in new_protocols]
+
+            if protocol_sync_method == 'append':
+                outdated_protocols = []
+
             for protocol in outdated_protocols:
                 scheme = protocol['scheme']
                 port = protocol['port']
@@ -67,11 +84,12 @@ def import_rses(rses, session=None):
             for protocol in missing_protocols:
                 rse_module.add_protocol(rse_id=rse_id, parameter=protocol, session=session)
 
-            for protocol in to_be_removed_protocols:
-                scheme = protocol['scheme']
-                port = protocol['port']
-                hostname = protocol['hostname']
-                rse_module.del_protocols(rse_id=rse_id, scheme=scheme, port=port, hostname=hostname, session=session)
+            if protocol_sync_method == 'hard':
+                for protocol in to_be_removed_protocols:
+                    scheme = protocol['scheme']
+                    port = protocol['port']
+                    hostname = protocol['hostname']
+                    rse_module.del_protocols(rse_id=rse_id, scheme=scheme, port=port, hostname=hostname, session=session)
 
         # Limits
         old_limits = rse_module.get_rse_limits(rse_id=rse_id, session=session)
@@ -88,21 +106,31 @@ def import_rses(rses, session=None):
         attributes['verify_checksum'] = rse.get('verify_checksum')
 
         old_attributes = rse_module.list_rse_attributes(rse_id=rse_id, session=session)
+        missing_attributes = [attribute for attribute in old_attributes if attribute not in attributes]
+
         for attr in attributes:
             value = attributes[attr]
             if value is not None:
                 if attr in old_attributes:
+                    if attr_sync_method not in ['append']:
+                        rse_module.del_rse_attribute(rse_id=rse_id, key=attr, session=session)
+                        rse_module.add_rse_attribute(rse_id=rse_id, key=attr, value=value, session=session)
+                else:
+                    rse_module.add_rse_attribute(rse_id=rse_id, key=attr, value=value, session=session)
+        if attr_sync_method == 'hard':
+            for attr in missing_attributes:
+                if attr != rse_name:
                     rse_module.del_rse_attribute(rse_id=rse_id, key=attr, session=session)
-                rse_module.add_rse_attribute(rse_id=rse_id, key=attr, value=value, session=session)
 
     # set deleted flag to RSEs that are missing in the import data
     old_rses = [old_rse['id'] for old_rse in rse_module.list_rses(session=session)]
-    for old_rse in old_rses:
-        if old_rse not in new_rses:
-            try:
-                rse_module.del_rse(rse_id=old_rse, session=session)
-            except RSEOperationNotSupported:
-                pass
+    if rse_sync_method == 'hard':
+        for old_rse in old_rses:
+            if old_rse not in new_rses:
+                try:
+                    rse_module.del_rse(rse_id=old_rse, session=session)
+                except RSEOperationNotSupported:
+                    pass
 
 
 @transactional_session
@@ -201,10 +229,13 @@ def import_data(data, session=None):
     :param data: data to be imported as dictionary.
     :param session: database session in use.
     """
-    # RSEs
+    rse_sync_method = config_get('importer', 'rse_sync_method', False, 'edit')
+    attr_sync_method = config_get('importer', 'attr_sync_method', False, 'edit')
+    protocol_sync_method = config_get('importer', 'rse_sync_method', False, 'edit')
+
     rses = data.get('rses')
     if rses:
-        import_rses(rses, session=session)
+        import_rses(rses, rse_sync_method=rse_sync_method, attr_sync_method=attr_sync_method, protocol_sync_method=protocol_sync_method, session=session)
 
     # Distances
     distances = data.get('distances')
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -27,6 +27,7 @@
 # - Gabriele Fronze' <gfronze@cern.ch>, 2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Brandon White <bjwhite@fnal.gov>, 2019
+# - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2019
 #
 # PY3K COMPATIBLE
 
@@ -119,7 +120,7 @@ def add_rse(rse, deterministic=True, volatile=False, city=None, region_code=None
 
 
 @read_session
-def rse_exists(rse, session=None):
+def rse_exists(rse, include_deleted=False, session=None):
     """
     Checks to see if RSE exists.
 
@@ -128,7 +129,7 @@ def rse_exists(rse, session=None):
 
     :returns: True if found, otherwise false.
     """
-    return True if session.query(models.RSE).filter_by(rse=rse, deleted=False).first() else False
+    return True if session.query(models.RSE).filter_by(rse=rse, deleted=include_deleted).first() else False
 
 
 @read_session
@@ -184,6 +185,27 @@ def del_rse(rse_id, session=None):
         pass
 
 
+@transactional_session
+def restore_rse(rse_id, session=None):
+    """
+    Restore a rse with the given rse id.
+
+    :param rse_id: the rse id.
+    :param session: The database session in use.
+    """
+
+    old_rse = None
+    try:
+        old_rse = session.query(models.RSE).filter_by(id=rse_id, deleted=True).one()
+    except sqlalchemy.orm.exc.NoResultFound:
+        raise exception.RSENotFound('RSE with id \'%s\' cannot be found' % rse_id)
+    old_rse.deleted = False
+    old_rse.deleted_at = None
+    old_rse.save(session=session)
+    rse = old_rse.rse
+    add_rse_attribute(rse_id=rse_id, key=rse, value=True, session=session)
+
+
 @read_session
 def rse_is_empty(rse_id, session=None):
     """


Test patch:
diff --git a/lib/rucio/tests/test_import_export.py b/lib/rucio/tests/test_import_export.py
--- a/lib/rucio/tests/test_import_export.py
+++ b/lib/rucio/tests/test_import_export.py
@@ -15,6 +15,7 @@
 # Authors:
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
+# - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2019
 #
 # PY3K COMPATIBLE
 
@@ -22,12 +23,14 @@
 
 from copy import deepcopy
 from nose.tools import assert_equal, assert_true, assert_raises, assert_in
+from nose import SkipTest
 from paste.fixture import TestApp
 
 from rucio.db.sqla import session, models
 from rucio.db.sqla.constants import RSEType, AccountType, IdentityType, AccountStatus
 from rucio.client.importclient import ImportClient
 from rucio.client.exportclient import ExportClient
+from rucio.common.config import config_set, config_add_section, config_has_section
 from rucio.common.exception import RSENotFound
 from rucio.common.types import InternalAccount
 from rucio.common.utils import render_json, parse_response
@@ -35,8 +38,8 @@
 from rucio.core.distance import add_distance, get_distances
 from rucio.core.exporter import export_data, export_rses
 from rucio.core.identity import add_identity, list_identities, add_account_identity, list_accounts_for_identity
-from rucio.core.importer import import_data
-from rucio.core.rse import get_rse_id, get_rse_name, add_rse, get_rse, add_protocol, get_rse_protocols, list_rse_attributes, get_rse_limits, set_rse_limits, add_rse_attribute, list_rses, export_rse, get_rse_attribute
+from rucio.core.importer import import_data, import_rses
+from rucio.core.rse import get_rse_id, get_rse_name, add_rse, get_rse, add_protocol, get_rse_protocols, list_rse_attributes, get_rse_limits, set_rse_limits, add_rse_attribute, list_rses, export_rse, get_rse_attribute, del_rse
 from rucio.tests.common import rse_name_generator
 from rucio.web.rest.importer import APP as import_app
 from rucio.web.rest.exporter import APP as export_app
@@ -83,9 +86,23 @@ def reset_rses():
     db_session.commit()
 
 
+def test_active():
+    db_session = session.get_session()
+    if db_session.bind.dialect.name == 'sqlite':
+        return False
+    return True
+
+
 class TestImporter(object):
+    """ Tests the initial import method (hard-sync everything) """
 
     def setup(self):
+        if not config_has_section('importer'):
+            config_add_section('importer')
+        config_set('importer', 'rse_sync_method', 'hard')
+        config_set('importer', 'attr_method', 'edit')
+        config_set('importer', 'protocol_method', 'edit')
+
         # New RSE
         self.new_rse = rse_name_generator()
 
@@ -371,8 +388,9 @@ def test_importer_client(self):
 
         self.check_accounts(self.data1['accounts'])
 
-        with assert_raises(RSENotFound):
-            get_rse(rse_id=self.old_rse_id_4)
+        # If the default sync method is not 'hard', RSE old_rse_id_4 should still be there
+        # with assert_raises(RSENotFound):
+        #     get_rse(rse_id=self.old_rse_id_4)
 
         import_client.import_data(data=self.data2)
         import_client.import_data(data=self.data3)
@@ -434,6 +452,739 @@ def test_importer_rest(self):
         assert_equal(r2.status, 201)
 
 
+class TestImporterSyncModes(object):
+
+    def setup(self):
+        # Since test config scenarios are complicated moved the setup inside the individual tests
+        if not test_active:
+            raise SkipTest
+
+    def test_import_rses_append(self):
+        """ IMPORTER (CORE): test import rse (APPEND mode). """
+        # In rse sync mode append: New RSEs are created, existing RSEs are not modified, leftover RSEs are not deleted
+
+        # RSE that did not exist before
+        new_rse = rse_name_generator()
+
+        # RSE missing from json
+        old_rse = rse_name_generator()
+        old_rse_id = add_rse(old_rse)
+
+        # RSE that was disabled but is active on json
+        disabled_rse = rse_name_generator()
+        disabled_rse_id = add_rse(disabled_rse)
+        del_rse(disabled_rse_id)
+
+        data = {
+            'rses': {
+                new_rse: {
+                    'rse_type': RSEType.TAPE,
+                    'availability': 3,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [{
+                        'scheme': 'scheme',
+                        'hostname': 'hostname',
+                        'port': 1000,
+                        'impl': 'impl'
+                    }],
+                    'attributes': {
+                        'attr1': 'test'
+                    },
+                    'MinFreeSpace': 20000,
+                    'lfn2pfn_algorithm': 'hash2',
+                    'verify_checksum': False,
+                    'availability_delete': True,
+                    'availability_read': False,
+                    'availability_write': True
+                },
+                disabled_rse: {
+                    'rse_type': RSEType.TAPE,
+                    'deterministic': True,
+                    'volatile': True,
+                    'region_code': 'DE',
+                    'country_name': 'DE',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'longitude': 2,
+                    'city': 'City',
+                    'availability': 1,
+                    'latitude': 1,
+                    'protocols': [{
+                        'scheme': 'scheme1',
+                        'hostname': 'hostname1',
+                        'port': 1000,
+                        'prefix': 'prefix',
+                        'impl': 'impl1'
+                    }, {
+                        'scheme': 'scheme2',
+                        'hostname': 'hostname2',
+                        'port': 1001,
+                        'impl': 'impl'
+                    }, {
+                        'scheme': 'scheme3',
+                        'hostname': 'hostname3',
+                        'port': 1001,
+                        'impl': 'impl'
+                    }],
+                    'attributes': {
+                        'attr1': 'test1',
+                        'attr2': 'test2'
+                    },
+                    'MinFreeSpace': 10000,
+                    'MaxBeingDeletedFiles': 1000,
+                    'verify_checksum': False,
+                    'lfn2pfn_algorithm': 'hash3',
+                    'availability_delete': False,
+                    'availability_read': False,
+                    'availability_write': True
+                }
+            }
+        }
+
+        import_rses(rses=deepcopy(data['rses']), rse_sync_method='append')
+
+        # Check RSE that did not exist before exists now
+        check_rse(new_rse, data['rses'])
+
+        # Check that old_rse was not disabled after import
+        assert_equal(get_rse_id(old_rse, include_deleted=False), old_rse_id)
+
+        # Check that disabled_rse dit not get enabled
+        with assert_raises(RSENotFound):
+            get_rse(rse_id=disabled_rse_id)
+
+    def test_import_rses_edit(self):
+        """ IMPORTER (CORE): test import rse (EDIT mode). """
+        # In rse sync mode edit: New RSEs are created, existing RSEs are modified, leftover RSEs are not deleted
+
+        # RSE that did not exist before
+        new_rse = rse_name_generator()
+
+        # RSE missing from json
+        old_rse = rse_name_generator()
+        old_rse_id = add_rse(old_rse)
+
+        # RSE that was disabled but is active on json
+        disabled_rse = rse_name_generator()
+        disabled_rse_id = add_rse(disabled_rse)
+        del_rse(disabled_rse_id)
+
+        data = {
+            'rses': {
+                new_rse: {
+                    'rse_type': RSEType.TAPE,
+                    'availability': 3,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [{
+                        'scheme': 'scheme',
+                        'hostname': 'hostname',
+                        'port': 1000,
+                        'impl': 'impl'
+                    }],
+                    'attributes': {
+                        'attr1': 'test'
+                    },
+                    'MinFreeSpace': 20000,
+                    'lfn2pfn_algorithm': 'hash2',
+                    'verify_checksum': False,
+                    'availability_delete': True,
+                    'availability_read': False,
+                    'availability_write': True
+                },
+                disabled_rse: {
+                    'rse_type': RSEType.TAPE,
+                    'deterministic': True,
+                    'volatile': True,
+                    'region_code': 'DE',
+                    'country_name': 'DE',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'longitude': 2,
+                    'city': 'City',
+                    'availability': 1,
+                    'latitude': 1,
+                    'protocols': [{
+                        'scheme': 'scheme1',
+                        'hostname': 'hostname1',
+                        'port': 1000,
+                        'prefix': 'prefix',
+                        'impl': 'impl1'
+                    }, {
+                        'scheme': 'scheme2',
+                        'hostname': 'hostname2',
+                        'port': 1001,
+                        'impl': 'impl'
+                    }, {
+                        'scheme': 'scheme3',
+                        'hostname': 'hostname3',
+                        'port': 1001,
+                        'impl': 'impl'
+                    }],
+                    'attributes': {
+                        'attr1': 'test1',
+                        'attr2': 'test2'
+                    },
+                    'MinFreeSpace': 10000,
+                    'MaxBeingDeletedFiles': 1000,
+                    'verify_checksum': False,
+                    'lfn2pfn_algorithm': 'hash3',
+                    'availability_delete': False,
+                    'availability_read': False,
+                    'availability_write': True
+                }
+            }
+        }
+
+        import_rses(rses=deepcopy(data['rses']), rse_sync_method='edit')
+
+        # Check RSE that did not exist before exists now
+        check_rse(new_rse, data['rses'])
+
+        # Check that old_rse was not disabled after import
+        assert_equal(get_rse_id(old_rse, include_deleted=False), old_rse_id)
+
+        # Check that disabled_rse got enabled
+        assert_equal(get_rse_id(disabled_rse, include_deleted=False), disabled_rse_id)
+
+    def test_import_attributes_append(self):
+        """ IMPORTER (CORE): test import attributes (APPEND mode). """
+        # In attributes sync mode append: New attributes are created, existing attributes are not modified, leftover attributes are not deleted
+
+        # RSE has less attributes than on json
+        less_attr_rse = rse_name_generator()
+        less_attr_rse_id = add_rse(less_attr_rse)
+        add_rse_attribute(rse_id=less_attr_rse_id, key='attr1', value='test')
+
+        # RSE has an attribute with different value
+        diff_attr_rse = rse_name_generator()
+        diff_attr_rse_id = add_rse(diff_attr_rse)
+        add_rse_attribute(rse_id=diff_attr_rse_id, key='attr1', value='test_original')
+        add_rse_attribute(rse_id=diff_attr_rse_id, key='attr2', value='test_original')
+
+        # RSE has attributes that are missing from the json
+        more_attr_rse = rse_name_generator()
+        more_attr_rse_id = add_rse(more_attr_rse)
+        add_rse_attribute(rse_id=more_attr_rse_id, key='attr1', value='test_original')
+        add_rse_attribute(rse_id=more_attr_rse_id, key='attr2', value='test_original')
+        add_rse_attribute(rse_id=more_attr_rse_id, key='attr3', value='test_original')
+
+        data = {
+            'rses': {
+                less_attr_rse: {
+                    'attributes': {
+                        'attr1': 'test',
+                        'attr2': 'test_new'
+
+                    }
+                },
+                diff_attr_rse: {
+                    'attributes': {
+                        'attr1': 'test_original_dif',
+                        'attr2': 'test_different'
+                    }
+                },
+                more_attr_rse: {
+                    'attributes': {
+                        'attr1': 'test_original',
+                        'attr2': 'test_original'
+                    }
+                }
+            }
+        }
+
+        import_rses(rses=deepcopy(data['rses']), rse_sync_method='edit', attr_sync_method='append')
+
+        # Check that attributes were added for less_attr_rse
+        assert_equal(get_rse_attribute('attr2', rse_id=less_attr_rse_id, use_cache=False), ['test_new'])
+
+        # Check that attributes were not modified for diff_attr_rse
+        assert_equal(get_rse_attribute('attr2', rse_id=diff_attr_rse_id, use_cache=False), ['test_original'])
+
+        # Check that attributes were missing from the json are not deleted
+        assert_equal(get_rse_attribute('attr3', rse_id=more_attr_rse_id, use_cache=False), ['test_original'])
+
+    def test_import_attributes_edit(self):
+        """ IMPORTER (CORE): test import attributes (EDIT mode). """
+        # In attributes sync mode edit: New attributes are created, existing attributes are modified, leftover attributes are not deleted
+
+        # RSE has less attributes than on json
+        less_attr_rse = rse_name_generator()
+        less_attr_rse_id = add_rse(less_attr_rse)
+        add_rse_attribute(rse_id=less_attr_rse_id, key='attr1', value='test')
+
+        # RSE has an attribute with different value
+        diff_attr_rse = rse_name_generator()
+        diff_attr_rse_id = add_rse(diff_attr_rse)
+        add_rse_attribute(rse_id=diff_attr_rse_id, key='attr1', value='test_original')
+        add_rse_attribute(rse_id=diff_attr_rse_id, key='attr2', value='test_original')
+
+        # RSE has attributes that are missing from the json
+        more_attr_rse = rse_name_generator()
+        more_attr_rse_id = add_rse(more_attr_rse)
+        add_rse_attribute(rse_id=more_attr_rse_id, key='attr1', value='test_original')
+        add_rse_attribute(rse_id=more_attr_rse_id, key='attr2', value='test_original')
+        add_rse_attribute(rse_id=more_attr_rse_id, key='attr3', value='test_original')
+
+        data = {
+            'rses': {
+                less_attr_rse: {
+                    'attributes': {
+                        'attr1': 'test',
+                        'attr2': 'test_new'
+
+                    }
+                },
+                diff_attr_rse: {
+                    'attributes': {
+                        'attr1': 'test_original_dif',
+                        'attr2': 'test_different'
+                    }
+                },
+                more_attr_rse: {
+                    'attributes': {
+                        'attr1': 'test_original',
+                        'attr2': 'test_original'
+                    }
+                }
+            }
+        }
+
+        import_rses(rses=deepcopy(data['rses']), rse_sync_method='edit', attr_sync_method='edit')
+
+        # Check that attributes were added for less_attr_rse
+        assert_equal(get_rse_attribute('attr2', rse_id=less_attr_rse_id, use_cache=False), ['test_new'])
+
+        # Check that attributes were modified for diff_attr_rse
+        assert_equal(get_rse_attribute('attr2', rse_id=diff_attr_rse_id, use_cache=False), ['test_different'])
+
+        # Check that attributes that were missing from the json are not deleted
+        assert_equal(get_rse_attribute('attr3', rse_id=more_attr_rse_id, use_cache=False), ['test_original'])
+
+    def test_import_attributes_hard(self):
+        """ IMPORTER (CORE): test import attributes (HARD mode). """
+        # In attributes sync mode hard: New attributes are created, existing attributes are modified, leftover attributes are deleted
+
+        # RSE has less attributes than on json
+        less_attr_rse = rse_name_generator()
+        less_attr_rse_id = add_rse(less_attr_rse)
+        add_rse_attribute(rse_id=less_attr_rse_id, key='attr1', value='test')
+
+        # RSE has an attribute with different value
+        diff_attr_rse = rse_name_generator()
+        diff_attr_rse_id = add_rse(diff_attr_rse)
+        add_rse_attribute(rse_id=diff_attr_rse_id, key='attr1', value='test_original')
+        add_rse_attribute(rse_id=diff_attr_rse_id, key='attr2', value='test_original')
+
+        # RSE has attributes that are missing from the json
+        more_attr_rse = rse_name_generator()
+        more_attr_rse_id = add_rse(more_attr_rse)
+        add_rse_attribute(rse_id=more_attr_rse_id, key='attr1', value='test_original')
+        add_rse_attribute(rse_id=more_attr_rse_id, key='attr2', value='test_original')
+        add_rse_attribute(rse_id=more_attr_rse_id, key='attr3', value='test_original')
+
+        data = {
+            'rses': {
+                less_attr_rse: {
+                    'attributes': {
+                        'attr1': 'test',
+                        'attr2': 'test_new'
+
+                    }
+                },
+                diff_attr_rse: {
+                    'attributes': {
+                        'attr1': 'test_original_dif',
+                        'attr2': 'test_different'
+                    }
+                },
+                more_attr_rse: {
+                    'attributes': {
+                        'attr1': 'test_original',
+                        'attr2': 'test_original'
+                    }
+                }
+            }
+        }
+
+        import_rses(rses=deepcopy(data['rses']), rse_sync_method='edit', attr_sync_method='hard')
+
+        # Check that attributes were added for less_attr_rse
+        assert_equal(get_rse_attribute('attr2', rse_id=less_attr_rse_id, use_cache=False), ['test_new'])
+
+        # Check that attributes were modified for diff_attr_rse
+        assert_equal(get_rse_attribute('attr2', rse_id=diff_attr_rse_id, use_cache=False), ['test_different'])
+
+        # Check that attributes that were missing from the json are deleted
+        assert_equal(get_rse_attribute('attr3', rse_id=more_attr_rse_id, use_cache=False), [])
+
+    def test_import_protocols_append(self):
+        """ IMPORTER (CORE): test import protocols (APPEND mode). """
+        # In protocols sync mode append: New protocols are created, existing protocols are not modified, leftover protocols are not deleted
+
+        less_prot_rse = rse_name_generator()
+        less_prot_rse_id = add_rse(less_prot_rse)
+        add_protocol(less_prot_rse_id, {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'TODO'})
+
+        diff_prot_rse = rse_name_generator()
+        diff_prot_rse_id = add_rse(diff_prot_rse)
+        add_protocol(diff_prot_rse_id, {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'TODO'})
+
+        more_prot_rse = rse_name_generator()
+        more_prot_rse_id = add_rse(more_prot_rse)
+        add_protocol(more_prot_rse_id, {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'TODO'})
+        add_protocol(more_prot_rse_id, {'scheme': 'scheme2', 'hostname': 'hostname2', 'port': 1000, 'impl': 'TODO'})
+        add_protocol(more_prot_rse_id, {'scheme': 'scheme3', 'hostname': 'hostname3', 'port': 1000, 'impl': 'TODO'})
+
+        data = {
+            'rses': {
+                less_prot_rse: {
+                    'rse_type': RSEType.TAPE,
+                    'availability': 3,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [
+                        {
+                            'scheme': 'scheme',
+                            'hostname': 'hostname',
+                            'port': 1000,
+                            'impl': 'impl'
+                        },
+                        {
+                            'scheme': 'scheme2',
+                            'hostname': 'hostname2',
+                            'port': 1000,
+                            'impl': 'impl'
+                        }
+                    ]
+                },
+                diff_prot_rse: {
+                    'rse_type': RSEType.DISK,
+                    'availability': 3,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [{
+                        'scheme': 'scheme',
+                        'hostname': 'hostname',
+                        'port': 1000,
+                        'impl': 'impl_new'
+                    }]
+                },
+                more_prot_rse: {
+                    'rse_type': RSEType.DISK,
+                    'availability': 2,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [
+                        {
+                            'scheme': 'scheme',
+                            'hostname': 'hostname',
+                            'port': 1000,
+                            'impl': 'impl'
+                        },
+                        {
+                            'scheme': 'scheme2',
+                            'hostname': 'hostname2',
+                            'port': 1000,
+                            'impl': 'impl'
+                        }
+                    ]
+                }
+            }
+        }
+
+        import_rses(rses=deepcopy(data['rses']), rse_sync_method='edit', protocol_sync_method='append')
+
+        # Check that new protocol was added
+        protocols = get_rse_protocols(less_prot_rse_id)
+        protocols_formated = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix']} for protocol in protocols['protocols']]
+        dp = data['rses'][less_prot_rse]['protocols'][0]
+        data_protocol_formated = {'hostname': dp['hostname'], 'scheme': dp['scheme'], 'port': dp['port'], 'impl': dp.get('impl', ''), 'prefix': dp.get('prefix', '')}
+        assert_in(data_protocol_formated, protocols_formated)
+
+        # Check that protocol was not modified
+        protocols = get_rse_protocols(diff_prot_rse_id)
+        protocols_formated = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl']} for protocol in protocols['protocols']]
+        data_protocol_formated = {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'TODO'}
+        assert_in(data_protocol_formated, protocols_formated)
+
+        # Check that missing protocol was not deleted
+        protocols = get_rse_protocols(more_prot_rse_id)
+        protocols_formated = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl']} for protocol in protocols['protocols']]
+        data_protocol_formated = {'scheme': 'scheme3', 'hostname': 'hostname3', 'port': 1000, 'impl': 'TODO'}
+        assert_in(data_protocol_formated, protocols_formated)
+
+    def test_import_protocols_edit(self):
+        """ IMPORTER (CORE): test import protocols (EDIT mode). """
+        # In protocols sync mode edit: New protocols are created, existing protocols are modified, leftover protocols are not deleted
+
+        less_prot_rse = rse_name_generator()
+        less_prot_rse_id = add_rse(less_prot_rse)
+        add_protocol(less_prot_rse_id, {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'TODO'})
+
+        diff_prot_rse = rse_name_generator()
+        diff_prot_rse_id = add_rse(diff_prot_rse)
+        add_protocol(diff_prot_rse_id, {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'TODO'})
+
+        more_prot_rse = rse_name_generator()
+        more_prot_rse_id = add_rse(more_prot_rse)
+        add_protocol(more_prot_rse_id, {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'TODO'})
+        add_protocol(more_prot_rse_id, {'scheme': 'scheme2', 'hostname': 'hostname2', 'port': 1000, 'impl': 'TODO'})
+        add_protocol(more_prot_rse_id, {'scheme': 'scheme3', 'hostname': 'hostname3', 'port': 1000, 'impl': 'TODO'})
+
+        data = {
+            'rses': {
+                less_prot_rse: {
+                    'rse_type': RSEType.TAPE,
+                    'availability': 3,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [
+                        {
+                            'scheme': 'scheme',
+                            'hostname': 'hostname',
+                            'port': 1000,
+                            'impl': 'impl'
+                        },
+                        {
+                            'scheme': 'scheme2',
+                            'hostname': 'hostname2',
+                            'port': 1000,
+                            'impl': 'impl'
+                        }
+                    ]
+                },
+                diff_prot_rse: {
+                    'rse_type': RSEType.DISK,
+                    'availability': 3,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [{
+                        'scheme': 'scheme1',
+                        'hostname': 'hostname1',
+                        'port': 1000,
+                        'impl': 'impl_new'
+                    }]
+                },
+                more_prot_rse: {
+                    'rse_type': RSEType.DISK,
+                    'availability': 2,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [
+                        {
+                            'scheme': 'scheme',
+                            'hostname': 'hostname',
+                            'port': 1000,
+                            'impl': 'impl'
+                        },
+                        {
+                            'scheme': 'scheme2',
+                            'hostname': 'hostname2',
+                            'port': 1000,
+                            'impl': 'impl'
+                        }
+                    ]
+                }
+            }
+        }
+
+        import_rses(rses=deepcopy(data['rses']), rse_sync_method='edit', protocol_sync_method='edit')
+
+        # Check that new protocol was added
+        protocols = get_rse_protocols(less_prot_rse_id)
+        protocols_formated = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix']} for protocol in protocols['protocols']]
+        dp = data['rses'][less_prot_rse]['protocols'][0]
+        data_protocol_formated = {'hostname': dp['hostname'], 'scheme': dp['scheme'], 'port': dp['port'], 'impl': dp.get('impl', ''), 'prefix': dp.get('prefix', '')}
+        assert_in(data_protocol_formated, protocols_formated)
+
+        # Check that protocol was added
+        protocols = get_rse_protocols(diff_prot_rse_id)
+        protocols_formated = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl']} for protocol in protocols['protocols']]
+        data_protocol_formated = {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'impl_new'}
+        assert_in(data_protocol_formated, protocols_formated)
+
+        # Check that missing protocol was not deleted
+        protocols = get_rse_protocols(more_prot_rse_id)
+        protocols_formated = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl']} for protocol in protocols['protocols']]
+        data_protocol_formated = {'scheme': 'scheme3', 'hostname': 'hostname3', 'port': 1000, 'impl': 'TODO'}
+        assert_in(data_protocol_formated, protocols_formated)
+
+    def test_import_protocols_hard(self):
+        """ IMPORTER (CORE): test import protocols (HARD mode). """
+        # In protocols sync mode hard: New protocols are created, existing protocols are modified, leftover protocols are deleted
+
+        less_prot_rse = rse_name_generator()
+        less_prot_rse_id = add_rse(less_prot_rse)
+        add_protocol(less_prot_rse_id, {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'TODO'})
+
+        diff_prot_rse = rse_name_generator()
+        diff_prot_rse_id = add_rse(diff_prot_rse)
+        add_protocol(diff_prot_rse_id, {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'TODO'})
+
+        more_prot_rse = rse_name_generator()
+        more_prot_rse_id = add_rse(more_prot_rse)
+        add_protocol(more_prot_rse_id, {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'TODO'})
+        add_protocol(more_prot_rse_id, {'scheme': 'scheme2', 'hostname': 'hostname2', 'port': 1000, 'impl': 'TODO'})
+        add_protocol(more_prot_rse_id, {'scheme': 'scheme3', 'hostname': 'hostname3', 'port': 1000, 'impl': 'TODO'})
+
+        data = {
+            'rses': {
+                less_prot_rse: {
+                    'rse_type': RSEType.TAPE,
+                    'availability': 3,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [
+                        {
+                            'scheme': 'scheme',
+                            'hostname': 'hostname',
+                            'port': 1000,
+                            'impl': 'impl'
+                        },
+                        {
+                            'scheme': 'scheme2',
+                            'hostname': 'hostname2',
+                            'port': 1000,
+                            'impl': 'impl'
+                        }
+                    ]
+                },
+                diff_prot_rse: {
+                    'rse_type': RSEType.DISK,
+                    'availability': 3,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [{
+                        'scheme': 'scheme1',
+                        'hostname': 'hostname1',
+                        'port': 1000,
+                        'impl': 'impl_new'
+                    }]
+                },
+                more_prot_rse: {
+                    'rse_type': RSEType.DISK,
+                    'availability': 2,
+                    'city': 'NewCity',
+                    'region_code': 'CH',
+                    'country_name': 'switzerland',
+                    'staging_area': False,
+                    'time_zone': 'Europe',
+                    'latitude': 1,
+                    'longitude': 2,
+                    'deterministic': True,
+                    'volatile': False,
+                    'protocols': [
+                        {
+                            'scheme': 'scheme',
+                            'hostname': 'hostname',
+                            'port': 1000,
+                            'impl': 'impl'
+                        },
+                        {
+                            'scheme': 'scheme2',
+                            'hostname': 'hostname2',
+                            'port': 1000,
+                            'impl': 'impl'
+                        }
+                    ]
+                }
+            }
+        }
+
+        import_rses(rses=deepcopy(data['rses']), rse_sync_method='edit', protocol_sync_method='hard')
+
+        # Check that new protocol was added
+        protocols = get_rse_protocols(less_prot_rse_id)
+        protocols_formated = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix']} for protocol in protocols['protocols']]
+        dp = data['rses'][less_prot_rse]['protocols'][0]
+        data_protocol_formated = {'hostname': dp['hostname'], 'scheme': dp['scheme'], 'port': dp['port'], 'impl': dp.get('impl', ''), 'prefix': dp.get('prefix', '')}
+        assert_in(data_protocol_formated, protocols_formated)
+
+        # Check that protocol was modified
+        protocols = get_rse_protocols(diff_prot_rse_id)
+        protocols_formated = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl']} for protocol in protocols['protocols']]
+        data_protocol_formated = {'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'impl': 'impl_new'}
+        assert_in(data_protocol_formated, protocols_formated)
+
+        # Check that missing protocol was deleted
+        protocols = get_rse_protocols(more_prot_rse_id)
+        protocols_formated = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl']} for protocol in protocols['protocols']]
+        data_protocol_formated = {'scheme': 'scheme3', 'hostname': 'hostname3', 'port': 1000, 'impl': 'TODO'}
+        assert(data_protocol_formated not in protocols_formated)
+
+
 class TestExporter(object):
 
     def setup(self):
@@ -500,6 +1251,12 @@ def tearDown(self):
 
     def test_export_import(self):
         """ IMPORT/EXPORT (REST): Test the export and import of data together to check same syntax."""
+        if not config_has_section('importer'):
+            config_add_section('importer')
+            config_set('importer', 'rse_sync_method', 'hard')
+            config_set('importer', 'attr_method', 'hard')
+            config_set('importer', 'protocol_method', 'hard')
+
         # Setup new RSE, distance, attribute, limits
         new_rse = rse_name_generator()
         add_rse(new_rse)
diff --git a/lib/rucio/tests/test_rse.py b/lib/rucio/tests/test_rse.py
--- a/lib/rucio/tests/test_rse.py
+++ b/lib/rucio/tests/test_rse.py
@@ -41,7 +41,7 @@
                                     InvalidObject, RSEProtocolDomainNotSupported, RSEProtocolPriorityError,
                                     ResourceTemporaryUnavailable, RSEAttributeNotFound, RSEOperationNotSupported)
 from rucio.common.utils import generate_uuid
-from rucio.core.rse import (add_rse, get_rse_id, del_rse, list_rses, rse_exists, add_rse_attribute, list_rse_attributes,
+from rucio.core.rse import (add_rse, get_rse_id, del_rse, restore_rse, list_rses, rse_exists, add_rse_attribute, list_rse_attributes,
                             set_rse_transfer_limits, get_rse_transfer_limits, delete_rse_transfer_limits,
                             get_rse_protocols, del_rse_attribute, get_rse_attribute, get_rse, rse_is_empty)
 from rucio.rse import rsemanager as mgr
@@ -180,6 +180,30 @@ def test_delete_rse(self):
         # with assert_raises(RSENotFound):
         #     del_rse(rse=rse_name)
 
+    def test_restore_rse(self):
+        """ RSE (CORE): Test restore of RSE """
+        # Restore deleted RSE
+        rse_name = rse_name_generator()
+        rse_id = add_rse(rse_name)
+        db_session = session.get_session()
+        db_session.commit()
+
+        del_rse(rse_id)
+        db_session.commit()
+        # Verify RSE was deleted
+        assert_equal(rse_exists(rse=rse_id), False)
+
+        restore_rse(rse_id=rse_id)
+        db_session.commit()
+        # Verify RSE was restored
+        assert rse_exists(rse=rse_name)
+
+        # Restoration of not deleted RSE:
+        rse_name = rse_name_generator()
+        rse_id = add_rse(rse_name)
+        with assert_raises(RSENotFound):
+            restore_rse(rse_id=rse_id)
+
     def test_empty_rse(self):
         """ RSE (CORE): Test if RSE is empty """
         rse_name = rse_name_generator()