Repository: rucio/rucio

Issue:
Rucio client package imports rucio.common.schema which is not part of distribution
Motivation
----------
From Isabel:
```
cmd: rucio upload --scope user.itrigger --rse TRIUMF-LCG2_SCRATCHDISK /tmp/itrigger/itrigger/.alrb/testing/tester.NKHUnK/rucio/bash/rucioUpload
2020-04-06 18:30:38,129
INFO Preparing upload for file myFile.d314ec0d-6684-4e3f-8501-47467586e335.log
2020-04-06 18:30:38,293
INFO Successfully added replica in Rucio catalogue at TRIUMF-LCG2_SCRATCHDISK
2020-04-06 18:30:38,842
INFO Successfully added replication rule at TRIUMF-LCG2_SCRATCHDISK
2020-04-06 18:30:39,962
ERROR No module named schema
2020-04-06 18:30:39,963
ERROR 
Rucio exited with an unexpected/unknown error.
Please rerun the last command with the "-v" option to gather more information.
If you're sure there is a problem with Rucio itself, please followup at: https://github.com/rucio/rucio/issues
rucio upload dataset (bash)                                         [FAILED]
```
For some reason the schema module seems to be imported, which is not part of the client distribution.

Modification
------------
We can add it to the distribution, but for me it is not clear why this is even needed, so we should understand this first.

add get_RSE_coverage_of_dataset function
Motivation
----------
to efficiently retrieve the amount of bytes of a dataset present on all RSEs


Modification
------------
to be added to rucio.core.replica




Discussion:
line where we import the schema in client mode:
https://github.com/rucio/rucio/blob/next/lib/rucio/rse/protocols/protocol.py#L230


Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -2960,3 +2960,34 @@ def set_tombstone(rse_id, scope, name, tombstone=OBSOLETE, session=None):
             raise exception.ReplicaIsLocked('Replica %s:%s on RSE %s is locked.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
         except NoResultFound:
             raise exception.ReplicaNotFound('Replica %s:%s on RSE %s could not be found.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
+
+
+@read_session
+def get_RSEcoverage_of_dataset(scope, name, session=None):
+    """
+    Get total bytes present on RSEs
+
+    :param scope:             Scope of the dataset
+    :param name:              Name of the dataset
+    :param session:           The db session.
+    :return:                  Dictionary { rse_id : <total bytes present at rse_id> }
+    """
+
+    query = session.query(models.RSEFileAssociation.rse_id, func.sum(models.DataIdentifierAssociation.bytes))
+
+    query = query.filter(and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
+                              models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
+                              models.DataIdentifierAssociation.scope == scope,
+                              models.DataIdentifierAssociation.name == name,
+                              models.RSEFileAssociation.state != ReplicaState.BEING_DELETED,
+                              ))
+
+    query = query.group_by(models.RSEFileAssociation.rse_id)
+
+    result = {}
+
+    for rse_id, total in query:
+        if total:
+            result[rse_id] = total
+
+    return result
diff --git a/setup_rucio_client.py b/setup_rucio_client.py
--- a/setup_rucio_client.py
+++ b/setup_rucio_client.py
@@ -14,7 +14,7 @@
 #
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2014-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2017-2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2017-2020
 
 import os
 import re
@@ -39,7 +39,7 @@
 COPY_ARGS = sys.argv[1:]
 NAME = 'rucio-clients'
 IS_RELEASE = False
-PACKAGES = ['rucio', 'rucio.client', 'rucio.common',
+PACKAGES = ['rucio', 'rucio.client', 'rucio.common', 'rucio.common.schema', 
             'rucio.rse.protocols', 'rucio.rse', 'rucio.tests']
 REQUIREMENTS_FILES = ['etc/pip-requires-client']
 DESCRIPTION = "Rucio Client Lite Package"


Test patch:
diff --git a/lib/rucio/tests/test_replica.py b/lib/rucio/tests/test_replica.py
--- a/lib/rucio/tests/test_replica.py
+++ b/lib/rucio/tests/test_replica.py
@@ -49,7 +49,7 @@
 from rucio.core.replica import (add_replica, add_replicas, delete_replicas, get_replicas_state,
                                 update_replica_lock_counter, get_replica, list_replicas,
                                 declare_bad_file_replicas, list_bad_replicas,
-                                update_replicas_paths, update_replica_state,
+                                update_replicas_paths, update_replica_state, get_RSEcoverage_of_dataset,
                                 get_replica_atime, touch_replica, get_bad_pfns, set_tombstone)
 from rucio.core.rse import add_rse, add_protocol, add_rse_attribute, del_rse_attribute, get_rse_id
 from rucio.client.ruleclient import RuleClient
@@ -537,6 +537,36 @@ def test_list_replicas_with_updated_after(self):
         assert_equal(len(list(list_replicas([{'scope': scope, 'name': dsn}], updated_after=t2))), 1)
         assert_equal(len(list(list_replicas([{'scope': scope, 'name': dsn}], updated_after=t3))), 0)
 
+    def test_get_RSE_coverage_of_dataset(self):
+        """ REPLICA (CORE): test RSE coverage retrieval """
+        scope = InternalScope('mock')
+        root = InternalAccount('root')
+        mock1 = get_rse_id(rse='MOCK')
+        mock3 = get_rse_id(rse='MOCK3')
+        mock4 = get_rse_id(rse='MOCK4')
+        dsn = 'ds_cov_test_%s' % generate_uuid()
+        add_did(scope=scope, name=dsn, type='DATASET', account=root)
+
+        # test empty dataset
+        cov = get_RSEcoverage_of_dataset(scope=scope, name=dsn)
+        print(cov)
+        assert_equal(cov, {})
+
+        # add files/replicas
+        for i in range(1, 8):
+            add_replica(rse_id=mock1, scope=scope, name=dsn + '_%06d.data' % i, bytes=100, account=root)
+        for i in range(8, 11):
+            add_replica(rse_id=mock3, scope=scope, name=dsn + '_%06d.data' % i, bytes=100, account=root)
+        for i in range(11, 16):
+            add_replica(rse_id=mock4, scope=scope, name=dsn + '_%06d.data' % i, bytes=100, account=root)
+
+        attach_dids(scope=scope, name=dsn, dids=[{'scope': scope, 'name': dsn + '_%06d.data' % i} for i in range(1, 16)], account=root)
+        cov = get_RSEcoverage_of_dataset(scope=scope, name=dsn)
+        print(cov)
+        assert_equal(cov[mock1], 700)
+        assert_equal(cov[mock3], 300)
+        assert_equal(cov[mock4], 500)
+
 
 class TestReplicaClients: