Repository: rucio/rucio

Issue:
Thread workload hashing is incorrect for MySQL and PostgreSQL
Motivation
----------
For both the MySQL and PostgreSQL database types, the hashing algorithm used to partition worker thread responsibilities has the divisor of the modulo operation being incorrectly computed, erroneously using total_threads-1 instead of the raw value of total threads. This has the consequence of preventing some threads from ever being assigned work. 

Modification
------------
I will provide a fix for this. I intend to create a function in db/sqla/util.py to do this, as this pattern for workload partitioning is repeated 21 times throughout the codebase.





Discussion:

Resolving patch:
diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py
--- a/lib/rucio/transfertool/fts3.py
+++ b/lib/rucio/transfertool/fts3.py
@@ -359,7 +359,7 @@ def bulk_query(self, transfer_ids, timeout=None):
         responses = {}
         fts_session = requests.Session()
         xfer_ids = ','.join(transfer_ids)
-        jobs = fts_session.get('%s/jobs/%s?files=file_state,dest_surl,finish_time,start_time,reason,source_surl,file_metadata' % (self.external_host, xfer_ids),
+        jobs = fts_session.get('%s/jobs/%s?files=file_state,dest_surl,finish_time,start_time,staging_start,staging_finished,reason,source_surl,file_metadata' % (self.external_host, xfer_ids),
                                verify=self.verify,
                                cert=self.cert,
                                headers={'Content-Type': 'application/json'},
@@ -700,6 +700,8 @@ def __format_new_response(self, fts_job_response, fts_files_response):
                                      'src_url': file_resp.get('source_surl', None),
                                      'dst_url': file_resp.get('dest_surl', None),
                                      'started_at': datetime.datetime.strptime(file_resp['start_time'], '%Y-%m-%dT%H:%M:%S') if file_resp['start_time'] else None,
+                                     'staging_start_at': datetime.datetime.strptime(file_resp['staging_start'], '%Y-%m-%dT%H:%M:%S') if file_resp['staging_start'] else None,
+                                     'staging_finished_at': datetime.datetime.strptime(file_resp['staging_finished'], '%Y-%m-%dT%H:%M:%S') if file_resp['staging_finished'] else None,
                                      'transferred_at': datetime.datetime.strptime(file_resp['finish_time'], '%Y-%m-%dT%H:%M:%S') if file_resp['finish_time'] else None,
                                      'duration': duration,
                                      'reason': file_resp.get('reason', None),
diff --git a/lib/rucio/vcsversion.py b/lib/rucio/vcsversion.py
--- a/lib/rucio/vcsversion.py
+++ b/lib/rucio/vcsversion.py
@@ -4,8 +4,8 @@
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '1.21.5',
-    'branch_nick': 'patch-0-Release__1_21_5_preparation',
-    'revision_id': '0ac16e69b21b8dae9195d6a52b3d04549ff1e471',
-    'revno': 9088
+    'version': '1.21.6',
+    'branch_nick': 'patch-0-Release__1_21_6_preparation',
+    'revision_id': '5f9e6db16019ae94518f6d7e591d611ceb925c03',
+    'revno': 9097
 }


Test patch: