Repository: rucio/rucio
Issue: check global quota in running RSESelector

Motivation
----------
check global quota not only when initializing the RSESelector but also when calling the select_rse method.


Modification
------------
update select_rse/__update_quota method in the RSESelector



Changed files:
lib/rucio/core/rse_selector.py, lib/rucio/tests/test_rse_selector.py

diff --git a/lib/rucio/core/rse_selector.py b/lib/rucio/core/rse_selector.py
index 3292d4e22..dbaedc606 100644
--- a/lib/rucio/core/rse_selector.py
+++ b/lib/rucio/core/rse_selector.py
@@ -94,7 +94,9 @@ class RSESelector():
                         local_quota_left = 0
                     else:
                         local_quota_left = quota_limit - get_usage(rse_id=rse['rse_id'], account=account, session=session)['bytes']
+
                     # check global quota
+                    rse['global_quota_left'] = {}
                     all_global_quota_enough = True
                     for rse_expression, limit in global_quota_limit.items():
                         if rse['rse_id'] in limit['resolved_rse_ids']:
@@ -110,6 +112,8 @@ class RSESelector():
                             if global_quota_left <= 0:
                                 all_global_quota_enough = False
                                 break
+                            else:
+                                rse['global_quota_left'][rse_expression] = global_quota_left
                     if local_quota_left > 0 and all_global_quota_enough:
                         rse['quota_left'] = local_quota_left
                         space_limit = get_rse_limits(name='MaxSpaceAvailable', rse_id=rse['rse_id'], session=session).get('MaxSpaceAvailable')
@@ -157,11 +161,25 @@ class RSESelector():
         if len(rses) < count:
             raise RSEOverQuota('There is insufficient space on any of the target RSE\\'s to fullfill the operation.')
 
-        # Remove rses which do not have enough quota
+        # Remove rses which do not have enough local quota
         rses = [rse for rse in rses if rse['quota_left'] > size]
         if len(rses) < count:
             raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\\'s to fullfill the operation.')
 
+        # Remove rses which do not have enough global quota
+        rses_with_enough_quota = []
+        for rse in rses:
+            enough_global_quota = True
+            for rse_expression in rse.get('global_quota_left', []):
+                if rse['global_quota_left'][rse_expression] < size:
+                    enough_global_quota = False
+                    break
+            if enough_global_quota:
+                rses_with_enough_quota.append(rse)
+        rses = rses_with_enough_quota
+        if len(rses) < count:
+            raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\\'s to fullfill the operation.')
+
         for copy in range(count):
             # Remove rses already in the result set
             rses = [rse for rse in rses if rse['rse_id'] not in [item[0] for item in result]]
@@ -202,6 +220,8 @@ class RSESelector():
         for element in self.rses:
             if element['rse_id'] == rse[0]:
                 element['quota_left'] -= size
+                for rse_expression in element.get('global_quota_left', []):
+                    element['global_quota_left'][rse_expression] -= size
                 return
 
     def __choose_rse(self, rses):
diff --git a/lib/rucio/tests/test_rse_selector.py b/lib/rucio/tests/test_rse_selector.py
index 28a2efc28..fbc5cb92c 100644
--- a/lib/rucio/tests/test_rse_selector.py
+++ b/lib/rucio/tests/test_rse_selector.py
@@ -120,3 +120,78 @@ class TestRSESelectorInit(object):
         set_global_account_limit(account=self.account, rse_expression=self.rse_2_name, bytes=10)
         rse_selector = RSESelector(self.account, rses, None, copies)
         assert_equal(len(rse_selector.rses), 1)
+
+
+class TestRSESelectorDynamic(object):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.account = InternalAccount('jdoe')
+        cls.rse_1_name = 'MOCK4'
+        cls.rse_2_name = 'MOCK5'
+        cls.mock1_id = get_rse_id(cls.rse_1_name)
+        cls.mock2_id = get_rse_id(cls.rse_2_name)
+        cls.db_session = session.get_session()
+        cls.rse_1 = {'id': cls.mock1_id, 'staging_area': False}
+        cls.rse_2 = {'id': cls.mock2_id, 'staging_area': False}
+
+    def setup(self):
+        self.db_session.query(models.AccountUsage).delete()
+        self.db_session.query(models.AccountLimit).delete()
+        self.db_session.query(models.AccountGlobalLimit).delete()
+        self.db_session.query(models.UpdatedAccountCounter).delete()
+        self.db_session.commit()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.db_session.query(models.AccountUsage).delete()
+        cls.db_session.query(models.AccountLimit).delete()
+        cls.db_session.query(models.AccountGlobalLimit).delete()
+        cls.db_session.query(models.UpdatedAccountCounter).delete()
+        cls.db_session.commit()
+        cls.db_session.close()
+
+    def test_1(self):
+        # enough RSEs and global quota, but not enough local quota after change -> 1 RSE
+        set_global_account_limit(account=self.account, rse_expression=self.rse_1_name, bytes=20)
+        set_global_account_limit(account=self.account, rse_expression=self.rse_2_name, bytes=20)
+        set_local_account_limit(account=self.account, rse_id=self.mock1_id, bytes=10)
+        set_local_account_limit(account=self.account, rse_id=self.mock2_id, bytes=10)
+        copies = 2
+        rses = [self.rse_1, self.rse_2]
+        rse_selector = RSESelector(self.account, rses, None, copies)
+        assert_equal(len(rse_selector.rses), 2)
+        rse_selector.select_rse(9, [self.mock1_id], copies=1)
+        rses = rse_selector.select_rse(5, [], copies=1)
+        assert_equal(len(rses), 1)
+        assert_equal(rses[0][0], self.mock2_id)
+
+    def test_2(self):
+        # enough RSEs and global quota, but not enough global quota after change -> 1 RSE
+        set_global_account_limit(account=self.account, rse_expression=self.rse_1_name, bytes=10)
+        set_global_account_limit(account=self.account, rse_expression=self.rse_2_name, bytes=10)
+        set_local_account_limit(account=self.account, rse_id=self.mock1_id, bytes=20)
+        set_local_account_limit(account=self.account, rse_id=self.mock2_id, bytes=20)
+        copies = 2
+        rses = [self.rse_1, self.rse_2]
+        rse_selector = RSESelector(self.account, rses, None, copies)
+        assert_equal(len(rse_selector.rses), 2)
+        rse_selector.select_rse(10, [self.mock1_id], copies=1)
+        rses = rse_selector.select_rse(5, [], copies=1)
+        assert_equal(len(rses), 1)
+        assert_equal(rses[0][0], self.mock2_id)
+
+    def test_3(self):
+        # enough RSEs and global quota, also after after change -> 2 RSE
+        set_global_account_limit(account=self.account, rse_expression=self.rse_1_name, bytes=20)
+        set_global_account_limit(account=self.account, rse_expression=self.rse_2_name, bytes=20)
+        set_local_account_limit(account=self.account, rse_id=self.mock1_id, bytes=20)
+        set_local_account_limit(account=self.account, rse_id=self.mock2_id, bytes=20)
+        copies = 2
+        rses = [self.rse_1, self.rse_2]
+        rse_selector = RSESelector(self.account, rses, None, copies)
+        assert_equal(len(rse_selector.rses), 2)
+        rse_selector.select_rse(10, [self.mock1_id], copies=1)
+        rse_selector.select_rse(10, [self.mock2_id], copies=1)
+        rses = rse_selector.select_rse(5, [], copies=2)
+        assert_equal(len(rses), 2)