Repository: rucio/rucio
Issue: Request priority ignored at initial job submission to transfertool 

Motivation
----------

`rucio update-rule --priority=5 8d8ab45e541741c1904cf18c100b249c` correctly sets the priority on rules and requests. That priority is correctly taken into consideration on job re-submissions after failures. However, the behavior is inconsistent at first submission, when the priority is completely ignored and substituted by a constant: https://github.com/rucio/rucio/blob/0fdb77adf110b4301dba7e971022232c8e1612cd/lib/rucio/daemons/conveyor/common.py#L341


Modification
------------

Ensure that conveyor-submitter correctly sets the priority. 


Changed files:
lib/rucio/tests/test_conveyor.py, lib/rucio/transfertool/fts3.py, lib/rucio/core/transfer.py, lib/rucio/daemons/conveyor/common.py

diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
index 51f9a9b2b..6b0d74b73 100644
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -183,7 +183,7 @@ class TransferDestination:
 
 class RequestWithSources:
     def __init__(self, id_, request_type, rule_id, scope, name, md5, adler32, byte_count, activity, attributes,
-                 previous_attempt_id, dest_rse_data, account, retry_count):
+                 previous_attempt_id, dest_rse_data, account, retry_count, priority):
 
         self.request_id = id_
         self.request_type = request_type
@@ -200,6 +200,7 @@ class RequestWithSources:
         self.dest_rse = dest_rse_data
         self.account = account
         self.retry_count = retry_count or 0
+        self.priority = priority if priority is not None else 3
 
         self.sources = []
 
@@ -574,7 +575,7 @@ def submit_bulk_transfers(external_host, transfers, transfertool='fts3', job_par
         logger(logging.DEBUG, 'job_files: %s' % transfers)
         transfer_id = GlobusTransferTool(external_host=None).bulk_submit(transfers=transfers, timeout=timeout)
     elif transfertool == 'mock':
-        transfer_id = MockTransfertool(external_host=None).submit(transfers, None)
+        transfer_id = MockTransfertool(external_host=None).submit(transfers, job_params)
     return transfer_id
 
 
@@ -1098,6 +1099,7 @@ def __create_transfer_definitions(
                     dest_rse_data=hop_dst_rse,
                     account=rws.account,
                     retry_count=0,
+                    priority=rws.priority,
                 ),
                 protocol_factory=protocol_factory,
             )
@@ -1665,7 +1667,8 @@ def __list_transfer_requests_and_source_replicas(
                                  models.Request.dest_rse_id,
                                  models.Request.retry_count,
                                  models.Request.account,
-                                 models.Request.created_at) \\
+                                 models.Request.created_at,
+                                 models.Request.priority) \\
         .with_hint(models.Request, "INDEX(REQUESTS REQUESTS_TYP_STA_UPD_IDX)", 'oracle') \\
         .filter(models.Request.state == request_state) \\
         .filter(models.Request.request_type == request_type) \\
@@ -1707,6 +1710,7 @@ def __list_transfer_requests_and_source_replicas(
                           sub_requests.c.dest_rse_id,
                           sub_requests.c.account,
                           sub_requests.c.retry_count,
+                          sub_requests.c.priority,
                           models.RSE.id.label("source_rse_id"),
                           models.RSE.rse,
                           models.RSEFileAssociation.path,
@@ -1738,7 +1742,7 @@ def __list_transfer_requests_and_source_replicas(
 
     requests_by_id = {}
     for (request_id, rule_id, scope, name, md5, adler32, byte_count, activity, attributes, previous_attempt_id, dest_rse_id, account, retry_count,
-         source_rse_id, source_rse_name, file_path, source_ranking, source_url, distance_ranking) in query:
+         priority, source_rse_id, source_rse_name, file_path, source_ranking, source_url, distance_ranking) in query:
 
         # rses (of unknown length) should be a temporary table to check against instead of this special case
         if rses and dest_rse_id not in rses:
@@ -1749,7 +1753,7 @@ def __list_transfer_requests_and_source_replicas(
             request = RequestWithSources(id_=request_id, request_type=request_type, rule_id=rule_id, scope=scope, name=name,
                                          md5=md5, adler32=adler32, byte_count=byte_count, activity=activity, attributes=attributes,
                                          previous_attempt_id=previous_attempt_id, dest_rse_data=RseData(id_=dest_rse_id),
-                                         account=account, retry_count=retry_count)
+                                         account=account, retry_count=retry_count, priority=priority)
             requests_by_id[request_id] = request
 
         if source_rse_id is not None:
diff --git a/lib/rucio/daemons/conveyor/common.py b/lib/rucio/daemons/conveyor/common.py
index 777b96ec7..ef7c4397f 100644
--- a/lib/rucio/daemons/conveyor/common.py
+++ b/lib/rucio/daemons/conveyor/common.py
@@ -209,7 +209,7 @@ def job_params_for_fts_transfer(transfer, bring_online, default_lifetime, archiv
                       'multi_sources': True if len(transfer.legacy_sources) > 1 else False,
                   },
                   'overwrite': overwrite,
-                  'priority': 3}
+                  'priority': transfer.rws.priority}
 
     if transfer.get('multihop', False):
         job_params['multihop'] = True
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
index c92911af8..e479f3297 100644
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -46,6 +46,7 @@ from rucio.db.sqla import models
 from rucio.db.sqla.constants import RequestState, RequestType, ReplicaState, RSEType
 from rucio.db.sqla.session import read_session, transactional_session
 from rucio.tests.common import skip_rse_tests_with_accounts
+from rucio.transfertool.fts3 import FTS3Transfertool
 
 MAX_POLL_WAIT_SECONDS = 60
 TEST_FTS_HOST = 'https://fts:8446'
@@ -455,7 +456,8 @@ def test_multihop_receiver_on_success(vo, did_factory, root_account, core_config
         all_rses = [src_rse_id, jump_rse_id, dst_rse_id]
 
         did = did_factory.upload_test_file(src_rse)
-        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+        rule_priority = 5
+        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None, priority=rule_priority)
         submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
 
         request = __wait_for_request_state(dst_rse_id=jump_rse_id, state=RequestState.DONE, run_poller=False, **did)
@@ -463,6 +465,9 @@ def test_multihop_receiver_on_success(vo, did_factory, root_account, core_config
         request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.DONE, run_poller=False, **did)
         assert request['state'] == RequestState.DONE
 
+        fts_response = FTS3Transfertool(external_host=TEST_FTS_HOST).bulk_query(request['external_id'])
+        assert fts_response[request['external_id']][request['id']]['priority'] == rule_priority
+
         # Two hops; both handled by receiver
         assert metrics_mock.get_sample_value('rucio_daemons_conveyor_receiver_update_request_state_total', labels={'updated': 'True'}) >= 2
     finally:
@@ -565,7 +570,7 @@ def test_stager(rse_factory, did_factory, root_account, replica_client):
     distance_core.add_distance(src_rse_id, dst_rse_id, ranking=10)
     rse_core.add_rse_attribute(src_rse_id, 'staging_buffer', dst_rse)
     for rse_id in all_rses:
-        rse_core.add_rse_attribute(rse_id, 'fts', 'https://fts:8446')
+        rse_core.add_rse_attribute(rse_id, 'fts', TEST_FTS_HOST)
 
     did = did_factory.upload_test_file(src_rse)
     replica = replica_core.get_replica(rse_id=src_rse_id, **did)
@@ -625,7 +630,7 @@ def test_overwrite_on_tape(rse_factory, did_factory, root_account, core_config_m
     distance_core.add_distance(rse2_id, rse3_id, ranking=10)
     rse_core.add_rse_attribute(rse2_id, 'available_for_multihop', True)
     for rse_id in all_rses:
-        rse_core.add_rse_attribute(rse_id, 'fts', 'https://fts:8446')
+        rse_core.add_rse_attribute(rse_id, 'fts', TEST_FTS_HOST)
 
     # multihop transfer:
     did1 = did_factory.upload_test_file(rse1)
diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py
index 2f60f9c90..0b3fc1abe 100644
--- a/lib/rucio/transfertool/fts3.py
+++ b/lib/rucio/transfertool/fts3.py
@@ -658,6 +658,7 @@ class FTS3Transfertool(Transfertool):
             resps[request_id] = {'new_state': None,
                                  'transfer_id': fts_job_response.get('job_id'),
                                  'job_state': fts_job_response.get('job_state', None),
+                                 'priority': fts_job_response.get('priority', None),
                                  'file_state': file_resp.get('file_state', None),
                                  'src_url': file_resp.get('source_surl', None),
                                  'dst_url': file_resp.get('dest_surl', None),