Repository: rucio/rucio

Issue:
add user data import 
Motivation
----------
add import of user data from CRIC


Modification
------------





Discussion:

Resolving patch:
diff --git a/lib/rucio/api/importer.py b/lib/rucio/api/importer.py
--- a/lib/rucio/api/importer.py
+++ b/lib/rucio/api/importer.py
@@ -14,6 +14,7 @@
 from rucio.api import permission
 from rucio.common import exception
 from rucio.common.schema import validate_schema
+from rucio.common.types import InternalAccount
 from rucio.core import importer
 
 
@@ -29,4 +30,6 @@ def import_data(data, issuer):
     if not permission.has_permission(issuer=issuer, action='import', kwargs=kwargs):
         raise exception.AccessDenied('Account %s can not import data' % issuer)
 
+    for account in data.get('accounts', []):
+        account['account'] = InternalAccount(account['account'])
     return importer.import_data(data)
diff --git a/lib/rucio/core/importer.py b/lib/rucio/core/importer.py
--- a/lib/rucio/core/importer.py
+++ b/lib/rucio/core/importer.py
@@ -20,8 +20,9 @@
 
 from six import string_types
 from rucio.common.exception import RSEOperationNotSupported, RSENotFound
-from rucio.core import rse as rse_module, distance as distance_module
-from rucio.db.sqla.constants import RSEType
+from rucio.core import rse as rse_module, distance as distance_module, account as account_module, identity as identity_module
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import RSEType, AccountType, IdentityType
 from rucio.db.sqla.session import transactional_session
 
 
@@ -127,6 +128,71 @@ def import_distances(distances, session=None):
                                              failed=distance.get('failed'), session=session)
 
 
+@transactional_session
+def import_identities(identities, account_name, old_identities, old_identity_account, account_email, session=None):
+    for identity in identities:
+        identity['type'] = IdentityType.from_sym(identity['type'])
+
+    missing_identities = [identity for identity in identities if (identity['identity'], identity['type']) not in old_identities]
+    missing_identity_account = [identity for identity in identities if (identity['identity'], identity['type'], account_name) not in old_identity_account]
+    to_be_removed_identity_account = [old_identity for old_identity in old_identity_account if (old_identity[0], old_identity[1], old_identity[2]) not in
+                                      [(identity['identity'], identity['type'], account_name) for identity in identities] and old_identity[2] == account_name]
+
+    # add missing identities
+    for identity in missing_identities:
+        identity_type = identity['type']
+        password = identity.get('password')
+        identity = identity['identity']
+        if identity_type == IdentityType.USERPASS:
+            identity_module.add_identity(identity=identity, password=password, email=account_email, type=identity_type, session=session)
+        elif identity_type == IdentityType.GSS or identity_type == IdentityType.SSH or identity_type == IdentityType.X509:
+            identity_module.add_identity(identity=identity, email=account_email, type=identity_type, session=session)
+
+    # add missing identity-account association
+    for identity in missing_identity_account:
+        identity_module.add_account_identity(identity['identity'], identity['type'], account_name, email=account_email, session=session)
+
+    # remove identities from account-identity association
+    for identity in to_be_removed_identity_account:
+        identity_module.del_account_identity(identity=identity[0], type=identity[1], account=identity[2], session=session)
+
+
+@transactional_session
+def import_accounts(accounts, session=None):
+    old_accounts = {account['account']: account for account in account_module.list_accounts(session=session)}
+    missing_accounts = [account for account in accounts if account['account'] not in old_accounts]
+    outdated_accounts = [account for account in accounts if account['account'] in old_accounts]
+    to_be_removed_accounts = [old_account for old_account in old_accounts if old_account not in [account['account'] for account in accounts]]
+    old_identities = identity_module.list_identities(session=session)
+    old_identity_account = session.query(models.IdentityAccountAssociation.identity, models.IdentityAccountAssociation.identity_type, models.IdentityAccountAssociation.account).all()
+
+    # add missing accounts
+    for account_dict in missing_accounts:
+        account = account_dict['account']
+        email = account_dict['email']
+        account_module.add_account(account=account, type=AccountType.USER, email=email, session=session)
+        identities = account_dict.get('identities', [])
+        if identities:
+            import_identities(identities, account, old_identities, old_identity_account, email, session=session)
+
+    # remove left over accounts
+    for account in to_be_removed_accounts:
+        if account.external != 'root':
+            account_module.del_account(account=account, session=session)
+
+    # update existing accounts
+    for account_dict in outdated_accounts:
+        account = account_dict['account']
+        email = account_dict['email']
+        old_account = old_accounts[account]
+        if email and old_account['email'] != email:
+            account_module.update_account(account, key='email', value=email, session=session)
+
+        identities = account_dict.get('identities', [])
+        if identities:
+            import_identities(identities, account, old_identities, old_identity_account, email, session=session)
+
+
 @transactional_session
 def import_data(data, session=None):
     """
@@ -144,3 +210,8 @@ def import_data(data, session=None):
     distances = data.get('distances')
     if distances:
         import_distances(distances, session=session)
+
+    # Accounts
+    accounts = data.get('accounts')
+    if accounts:
+        import_accounts(accounts, session=session)


Test patch:
diff --git a/lib/rucio/tests/test_import_export.py b/lib/rucio/tests/test_import_export.py
--- a/lib/rucio/tests/test_import_export.py
+++ b/lib/rucio/tests/test_import_export.py
@@ -21,17 +21,20 @@
 from __future__ import print_function
 
 from copy import deepcopy
-from nose.tools import assert_equal, assert_true, assert_raises
+from nose.tools import assert_equal, assert_true, assert_raises, assert_in
 from paste.fixture import TestApp
 
 from rucio.db.sqla import session, models
-from rucio.db.sqla.constants import RSEType
+from rucio.db.sqla.constants import RSEType, AccountType, IdentityType, AccountStatus
 from rucio.client.importclient import ImportClient
 from rucio.client.exportclient import ExportClient
 from rucio.common.exception import RSENotFound
+from rucio.common.types import InternalAccount
 from rucio.common.utils import render_json, parse_response
+from rucio.core.account import add_account, get_account
 from rucio.core.distance import add_distance, get_distances
 from rucio.core.exporter import export_data, export_rses
+from rucio.core.identity import add_identity, list_identities, add_account_identity, list_accounts_for_identity
 from rucio.core.importer import import_data
 from rucio.core.rse import get_rse_id, get_rse_name, add_rse, get_rse, add_protocol, get_rse_protocols, list_rse_attributes, get_rse_limits, set_rse_limits, add_rse_attribute, list_rses, export_rse, get_rse_attribute
 from rucio.tests.common import rse_name_generator
@@ -118,6 +121,23 @@ def setup(self):
         # Distance that already exists
         add_distance(self.old_rse_id_1, self.old_rse_id_2)
 
+        # Account 1 that already exists
+        self.old_account_1 = InternalAccount(rse_name_generator())
+        add_account(self.old_account_1, AccountType.USER, email='test')
+
+        # Account 2 that already exists
+        self.old_account_2 = InternalAccount(rse_name_generator())
+        add_account(self.old_account_2, AccountType.USER, email='test')
+
+        # Identity that should be removed
+        self.identity_to_be_removed = rse_name_generator()
+        add_identity(self.identity_to_be_removed, IdentityType.X509, email='email')
+        add_account_identity(self.identity_to_be_removed, IdentityType.X509, self.old_account_2, 'email')
+
+        # Identity that already exsits but should be added to the account
+        self.identity_to_be_added_to_account = rse_name_generator()
+        add_identity(self.identity_to_be_added_to_account, IdentityType.X509, email='email')
+
         self.data1 = {
             'rses': {
                 self.new_rse: {
@@ -192,7 +212,36 @@ def setup(self):
                     self.old_rse_2: {'src_rse': self.old_rse_1, 'dest_rse': self.old_rse_2, 'ranking': 10},
                     self.old_rse_3: {'src_rse': self.old_rse_1, 'dest_rse': self.old_rse_3, 'ranking': 4}
                 }
-            }
+            },
+            'accounts': [{
+                'account': InternalAccount('new_account'),
+                'email': 'email',
+                'identities': [{
+                    'type': 'userpass',
+                    'identity': 'username',
+                    'password': 'password'
+                }]
+            }, {
+                'account': InternalAccount('new_account2'),
+                'email': 'email'
+            }, {
+                'account': self.old_account_2,
+                'email': 'new_email',
+                'identities': [
+                    {
+                        'identity': self.identity_to_be_added_to_account,
+                        'type': 'x509'
+                    },
+                    {
+                        'type': 'userpass',
+                        'identity': 'username2',
+                        'password': 'password'
+                    }
+                ]
+            }, {
+                'account': InternalAccount('jdoe'),
+                'email': 'email'
+            }]
         }
 
         self.data2 = {'rses': {self.new_rse: {'rse': self.new_rse}}}
@@ -201,6 +250,37 @@ def setup(self):
     def tearDown(self):
         reset_rses()
 
+    def check_accounts(self, test_accounts):
+        db_identities = list_identities()
+        for account in test_accounts:
+            # check existence
+            db_account = get_account(account=account['account'])
+            assert_equal(db_account['account'], account['account'])
+
+            # check properties
+            email = account.get('email')
+            if email:
+                assert_equal(db_account['email'], account['email'])
+
+            # check identities
+            identities = account.get('identities')
+            if identities:
+                for identity in identities:
+                    # check identity creation and identity-account association
+                    identity_type = IdentityType.from_sym(identity['type'])
+                    identity = identity['identity']
+                    assert_in((identity, identity_type), db_identities)
+                    accounts_for_identity = list_accounts_for_identity(identity, identity_type)
+                    assert_in(account['account'], accounts_for_identity)
+
+        # check removal of account
+        account = get_account(self.old_account_1)
+        assert_equal(account['status'], AccountStatus.DELETED)
+
+        # check removal of identities
+        accounts_for_identity = list_accounts_for_identity(self.identity_to_be_removed, IdentityType.X509)
+        assert_true(account['account'] not in accounts_for_identity)
+
     def test_importer_core(self):
         """ IMPORTER (CORE): test import. """
         import_data(data=deepcopy(self.data1))
@@ -241,6 +321,8 @@ def test_importer_core(self):
         distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
         assert_equal(distance['ranking'], 4)
 
+        self.check_accounts(self.data1['accounts'])
+
         # RSE 4 should be flagged as deleted as it is missing in the import data
         with assert_raises(RSENotFound):
             get_rse(rse_id=self.old_rse_id_4)
@@ -287,6 +369,8 @@ def test_importer_client(self):
         distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
         assert_equal(distance['ranking'], 4)
 
+        self.check_accounts(self.data1['accounts'])
+
         with assert_raises(RSENotFound):
             get_rse(rse_id=self.old_rse_id_4)
 
@@ -338,6 +422,8 @@ def test_importer_rest(self):
         distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
         assert_equal(distance['ranking'], 4)
 
+        self.check_accounts(self.data1['accounts'])
+
         with assert_raises(RSENotFound):
             get_rse(rse_id=self.old_rse_id_4)