Repository: rucio/rucio

Issue:
core.account_limit.get_rse_account_usage should liste accounts who have usage, but no quota
Motivation
----------
Due to the join of the two tables, only accounts with usage and quota are listed, however, accounts without quota should be listed as well.
Regression in bulk_group_transfer due to missing key
Motivation
----------
Due to the new verify_checksum mapping.
```
2018-04-24 14:36:42,467 2991    INFO    0:0 Got 1 transfers for Functional Test XrootD
2018-04-24 14:36:42,468 2991    INFO    0:0 Starting to group transfers for Functional Test XrootD
2018-04-24 14:36:42,468 2991    CRITICAL        0:0 Traceback (most recent call last):
File “/usr/lib/python2.7/site-packages/rucio/daemons/conveyor/submitter.py”, line 157, in submitter
  grouped_jobs = bulk_group_transfer(transfers, group_policy, group_bulk, fts_source_strategy, max_time_in_queue)
File “/usr/lib/python2.7/site-packages/rucio/daemons/conveyor/common.py”, line 156, in bulk_group_transfer
  ‘verify_checksum’: transfer[‘verify_checksum’],
KeyError: ‘verify_checksum’
```


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/account_limit.py b/lib/rucio/core/account_limit.py
--- a/lib/rucio/core/account_limit.py
+++ b/lib/rucio/core/account_limit.py
@@ -1,16 +1,21 @@
-'''
-  Copyright European Organization for Nuclear Research (CERN)
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  You may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-  http://www.apache.org/licenses/LICENSE-2.0
-
-  Authors:
-  - Martin Barisits, <martin.barisits@cern.ch>, 2013-2015
-  - Cedric Serfon, <cedric.serfon@cern.ch>, 2015
-  - Vincent Garonne, <vincent.garonne@cern.ch>, 2015
-'''
+# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+#  - Martin Barisits, <martin.barisits@cern.ch>, 2013-2018
+#  - Cedric Serfon, <cedric.serfon@cern.ch>, 2015
+#  - Vincent Garonne, <vincent.garonne@cern.ch>, 2015
 
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql.expression import and_, or_
@@ -32,7 +37,7 @@ def get_rse_account_usage(rse, session=None):
     result = []
     rse_id = get_rse_id(rse=rse, session=session)
     query = session.query(models.AccountUsage.account, models.AccountUsage.files, models.AccountUsage.bytes, models.AccountLimit.bytes)
-    query = query.join(models.AccountLimit, and_(models.AccountUsage.account == models.AccountLimit.account, models.AccountUsage.rse_id == models.AccountLimit.rse_id)).filter(models.AccountUsage.rse_id == rse_id)
+    query = query.outerjoin(models.AccountLimit, and_(models.AccountUsage.account == models.AccountLimit.account, models.AccountUsage.rse_id == models.AccountLimit.rse_id)).filter(models.AccountUsage.rse_id == rse_id)
     account_limits_tmp = query.all()
     for row in account_limits_tmp:
         result.append({'rse': rse, 'account': row[0], 'used_files': row[1], 'used_bytes': row[2], 'quota_bytes': row[3]})
diff --git a/lib/rucio/daemons/conveyor/common.py b/lib/rucio/daemons/conveyor/common.py
--- a/lib/rucio/daemons/conveyor/common.py
+++ b/lib/rucio/daemons/conveyor/common.py
@@ -148,12 +148,13 @@ def bulk_group_transfer(transfers, policy='rule', group_bulk=200, fts_source_str
             grouped_transfers[external_host] = {}
             grouped_jobs[external_host] = []
 
+        verify_checksum = transfer.get('verify_checksum', 'both')
         file = {'sources': transfer['sources'],
                 'destinations': transfer['dest_urls'],
                 'metadata': transfer['file_metadata'],
                 'filesize': int(transfer['file_metadata']['filesize']),
                 'checksum': None,
-                'verify_checksum': transfer['verify_checksum'],
+                'verify_checksum': verify_checksum,
                 'selection_strategy': fts_source_strategy,
                 'request_type': transfer['file_metadata'].get('request_type', None),
                 'activity': str(transfer['file_metadata']['activity'])}
@@ -163,7 +164,7 @@ def bulk_group_transfer(transfers, policy='rule', group_bulk=200, fts_source_str
             if 'adler32' in list(file['metadata'].keys()) and file['metadata']['adler32']:
                 file['checksum'] = 'ADLER32:%s' % str(file['metadata']['adler32'])
 
-        job_params = {'verify_checksum': transfer['verify_checksum'],
+        job_params = {'verify_checksum': verify_checksum,
                       'copy_pin_lifetime': transfer['copy_pin_lifetime'] if transfer['copy_pin_lifetime'] else -1,
                       'bring_online': transfer['bring_online'] if transfer['bring_online'] else None,
                       'job_metadata': {'issuer': 'rucio'},  # finaly job_meta will like this. currently job_meta will equal file_meta to include request_id and etc.


Test patch: