Repository: rucio/rucio

Issue:
Conveyor-submitter optional partitioning on `rule_id`
Motivation
----------
Currently the partitioning is done on request_id, which has the effect that transfers of the same rule are often not bulked together. Changing to `rule_id` should solve that, but comes with the risk of starvation issues.

Modification
------------
Make it an option to partition by `rule_id`



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -1166,13 +1166,14 @@ def __transfer_order_key(transfer_path):
 
 
 @transactional_session
-def next_transfers_to_submit(total_workers=0, worker_number=0, limit=None, activity=None, older_than=None, rses=None, schemes=None,
+def next_transfers_to_submit(total_workers=0, worker_number=0, partition_hash_var=None, limit=None, activity=None, older_than=None, rses=None, schemes=None,
                              failover_schemes=None, filter_transfertool=None, transfertools_by_name=None, request_type=RequestType.TRANSFER,
                              logger=logging.log, session=None):
     """
     Get next transfers to be submitted; grouped by transfertool which can submit them
     :param total_workers:         Number of total workers.
     :param worker_number:         Id of the executing worker.
+    :param partition_hash_var     The hash variable used for partitioning thread work
     :param limit:                 Maximum number of requests to retrieve from database.
     :param activity:              Activity.
     :param older_than:            Get transfers older than.
@@ -1210,6 +1211,7 @@ def next_transfers_to_submit(total_workers=0, worker_number=0, limit=None, activ
     request_with_sources = __list_transfer_requests_and_source_replicas(
         total_workers=total_workers,
         worker_number=worker_number,
+        partition_hash_var=partition_hash_var,
         limit=limit,
         activity=activity,
         older_than=older_than,
@@ -1554,6 +1556,7 @@ def create_missing_replicas_and_requests(
 def __list_transfer_requests_and_source_replicas(
     total_workers=0,
     worker_number=0,
+    partition_hash_var=None,
     limit=None,
     activity=None,
     older_than=None,
@@ -1565,19 +1568,23 @@ def __list_transfer_requests_and_source_replicas(
 ) -> "List[RequestWithSources]":
     """
     List requests with source replicas
-    :param total_workers:    Number of total workers.
-    :param worker_number:    Id of the executing worker.
-    :param limit:            Integer of requests to retrieve.
-    :param activity:         Activity to be selected.
-    :param older_than:       Only select requests older than this DateTime.
-    :param rses:             List of rse_id to select requests.
-    :param request_type:     Filter on the given request type.
-    :param request_state:    Filter on the given request state
-    :param transfertool:     The transfer tool as specified in rucio.cfg.
-    :param session:          Database session to use.
-    :returns:                List of RequestWithSources objects.
+    :param total_workers:     Number of total workers.
+    :param worker_number:     Id of the executing worker.
+    :param partition_hash_var The hash variable used for partitioning thread work
+    :param limit:             Integer of requests to retrieve.
+    :param activity:          Activity to be selected.
+    :param older_than:        Only select requests older than this DateTime.
+    :param rses:              List of rse_id to select requests.
+    :param request_type:      Filter on the given request type.
+    :param request_state:     Filter on the given request state
+    :param transfertool:      The transfer tool as specified in rucio.cfg.
+    :param session:           Database session to use.
+    :returns:                 List of RequestWithSources objects.
     """
 
+    if partition_hash_var is None:
+        partition_hash_var = 'request.id'
+
     if request_state is None:
         request_state = RequestState.QUEUED
 
@@ -1618,7 +1625,7 @@ def __list_transfer_requests_and_source_replicas(
     else:
         sub_requests = sub_requests.with_hint(models.Request, "INDEX(REQUESTS REQUESTS_TYP_STA_UPD_IDX)", 'oracle')
 
-    sub_requests = filter_thread_work(session=session, query=sub_requests, total_threads=total_workers, thread_id=worker_number, hash_variable='requests.id')
+    sub_requests = filter_thread_work(session=session, query=sub_requests, total_threads=total_workers, thread_id=worker_number, hash_variable=partition_hash_var)
 
     if limit:
         sub_requests = sub_requests.limit(limit)
diff --git a/lib/rucio/daemons/conveyor/submitter.py b/lib/rucio/daemons/conveyor/submitter.py
--- a/lib/rucio/daemons/conveyor/submitter.py
+++ b/lib/rucio/daemons/conveyor/submitter.py
@@ -83,6 +83,10 @@ def submitter(once=False, rses=None, partition_wait_time=10,
     Main loop to submit a new transfer primitive to a transfertool.
     """
 
+    try:
+        partition_hash_var = config_get('conveyor', 'partition_hash_var')
+    except NoOptionError:
+        partition_hash_var = None
     try:
         scheme = config_get('conveyor', 'scheme')
     except NoOptionError:
@@ -167,6 +171,7 @@ def submitter(once=False, rses=None, partition_wait_time=10,
                     transfers = transfer_core.next_transfers_to_submit(
                         total_workers=heart_beat['nr_threads'],
                         worker_number=heart_beat['assign_thread'],
+                        partition_hash_var=partition_hash_var,
                         failover_schemes=failover_scheme,
                         limit=bulk,
                         activity=activity,


Test patch: