Repository: rucio/rucio

Issue:
add get_RSE_coverage_of_dataset function
Motivation
----------
to efficiently retrieve the amount of bytes of a dataset present on all RSEs


Modification
------------
to be added to rucio.core.replica




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -2960,3 +2960,34 @@ def set_tombstone(rse_id, scope, name, tombstone=OBSOLETE, session=None):
             raise exception.ReplicaIsLocked('Replica %s:%s on RSE %s is locked.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
         except NoResultFound:
             raise exception.ReplicaNotFound('Replica %s:%s on RSE %s could not be found.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
+
+
+@read_session
+def get_RSEcoverage_of_dataset(scope, name, session=None):
+    """
+    Get total bytes present on RSEs
+
+    :param scope:             Scope of the dataset
+    :param name:              Name of the dataset
+    :param session:           The db session.
+    :return:                  Dictionary { rse_id : <total bytes present at rse_id> }
+    """
+
+    query = session.query(models.RSEFileAssociation.rse_id, func.sum(models.DataIdentifierAssociation.bytes))
+
+    query = query.filter(and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
+                              models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
+                              models.DataIdentifierAssociation.scope == scope,
+                              models.DataIdentifierAssociation.name == name,
+                              models.RSEFileAssociation.state != ReplicaState.BEING_DELETED,
+                              ))
+
+    query = query.group_by(models.RSEFileAssociation.rse_id)
+
+    result = {}
+
+    for rse_id, total in query:
+        if total:
+            result[rse_id] = total
+
+    return result


Test patch:
diff --git a/lib/rucio/tests/test_replica.py b/lib/rucio/tests/test_replica.py
--- a/lib/rucio/tests/test_replica.py
+++ b/lib/rucio/tests/test_replica.py
@@ -49,7 +49,7 @@
 from rucio.core.replica import (add_replica, add_replicas, delete_replicas, get_replicas_state,
                                 update_replica_lock_counter, get_replica, list_replicas,
                                 declare_bad_file_replicas, list_bad_replicas,
-                                update_replicas_paths, update_replica_state,
+                                update_replicas_paths, update_replica_state, get_RSEcoverage_of_dataset,
                                 get_replica_atime, touch_replica, get_bad_pfns, set_tombstone)
 from rucio.core.rse import add_rse, add_protocol, add_rse_attribute, del_rse_attribute, get_rse_id
 from rucio.client.ruleclient import RuleClient
@@ -537,6 +537,36 @@ def test_list_replicas_with_updated_after(self):
         assert_equal(len(list(list_replicas([{'scope': scope, 'name': dsn}], updated_after=t2))), 1)
         assert_equal(len(list(list_replicas([{'scope': scope, 'name': dsn}], updated_after=t3))), 0)
 
+    def test_get_RSE_coverage_of_dataset(self):
+        """ REPLICA (CORE): test RSE coverage retrieval """
+        scope = InternalScope('mock')
+        root = InternalAccount('root')
+        mock1 = get_rse_id(rse='MOCK')
+        mock3 = get_rse_id(rse='MOCK3')
+        mock4 = get_rse_id(rse='MOCK4')
+        dsn = 'ds_cov_test_%s' % generate_uuid()
+        add_did(scope=scope, name=dsn, type='DATASET', account=root)
+
+        # test empty dataset
+        cov = get_RSEcoverage_of_dataset(scope=scope, name=dsn)
+        print(cov)
+        assert_equal(cov, {})
+
+        # add files/replicas
+        for i in range(1, 8):
+            add_replica(rse_id=mock1, scope=scope, name=dsn + '_%06d.data' % i, bytes=100, account=root)
+        for i in range(8, 11):
+            add_replica(rse_id=mock3, scope=scope, name=dsn + '_%06d.data' % i, bytes=100, account=root)
+        for i in range(11, 16):
+            add_replica(rse_id=mock4, scope=scope, name=dsn + '_%06d.data' % i, bytes=100, account=root)
+
+        attach_dids(scope=scope, name=dsn, dids=[{'scope': scope, 'name': dsn + '_%06d.data' % i} for i in range(1, 16)], account=root)
+        cov = get_RSEcoverage_of_dataset(scope=scope, name=dsn)
+        print(cov)
+        assert_equal(cov[mock1], 700)
+        assert_equal(cov[mock3], 300)
+        assert_equal(cov[mock4], 500)
+
 
 class TestReplicaClients: