Repository: rucio/rucio

Issue:
Remove full_mode switch from conveyor-receiver
Motivation
----------
Since this works well now, everything should be full_mode 😄 


Modification
------------





Discussion:
Blocked by #5532 

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -30,7 +30,7 @@
 from rucio.common.constants import SUPPORTED_PROTOCOLS
 from rucio.common.exception import (InvalidRSEExpression,
                                     RequestNotFound, RSEProtocolNotSupported,
-                                    RucioException, UnsupportedOperation)
+                                    RucioException)
 from rucio.common.utils import construct_surl
 from rucio.core import did, message as message_core, request as request_core
 from rucio.core.account import list_accounts
@@ -508,36 +508,6 @@ def mark_transfer_lost(request, *, session: "Session", logger=logging.log):
     request_core.add_monitor_message(new_state=new_state, request=request, additional_fields={'reason': reason}, session=session)
 
 
-@METRICS.count_it
-@transactional_session
-def set_transfer_update_time(external_host, transfer_id, update_time=datetime.datetime.utcnow(), *, session: "Session"):
-    """
-    Update the state of a request. Fails silently if the transfer_id does not exist.
-    :param external_host:  Selected external host as string in format protocol://fqdn:port
-    :param transfer_id:    External transfer job id as a string.
-    :param update_time:    Time stamp.
-    :param session:        Database session to use.
-    """
-
-    try:
-        stmt = update(
-            models.Request
-        ).where(
-            models.Request.external_id == transfer_id,
-            models.Request.state == RequestState.SUBMITTED
-        ).execution_options(
-            synchronize_session=False
-        ).values(
-            updated_at=update_time
-        )
-        rowcount = session.execute(stmt).rowcount
-    except IntegrityError as error:
-        raise RucioException(error.args)
-
-    if not rowcount:
-        raise UnsupportedOperation("Transfer %s doesn't exist or its status is not submitted." % transfer_id)
-
-
 @METRICS.count_it
 @transactional_session
 def touch_transfer(external_host, transfer_id, *, session: "Session"):
diff --git a/lib/rucio/daemons/conveyor/receiver.py b/lib/rucio/daemons/conveyor/receiver.py
--- a/lib/rucio/daemons/conveyor/receiver.py
+++ b/lib/rucio/daemons/conveyor/receiver.py
@@ -17,7 +17,6 @@
 Conveyor is a daemon to manage file transfers.
 """
 
-import datetime
 import json
 import logging
 import socket
@@ -34,7 +33,6 @@
 from rucio.common.policy import get_policy
 from rucio.core import request as request_core
 from rucio.core.monitor import MetricManager
-from rucio.core.transfer import set_transfer_update_time
 from rucio.daemons.common import HeartbeatHandler
 from rucio.db.sqla.session import transactional_session
 from rucio.transfertool.fts3 import FTS3CompletionMessageTransferStatusReport
@@ -47,12 +45,11 @@
 
 class Receiver(object):
 
-    def __init__(self, broker, id_, total_threads, full_mode=False, all_vos=False):
+    def __init__(self, broker, id_, total_threads, all_vos=False):
         self.__all_vos = all_vos
         self.__broker = broker
         self.__id = id_
         self.__total_threads = total_threads
-        self.__full_mode = full_mode
 
     @METRICS.count_it
     def on_error(self, frame):
@@ -85,26 +82,18 @@ def _perform_request_update(self, msg, *, session=None, logger=logging.log):
             if tt_status_report.get_db_fields_to_update(session=session, logger=logger):
                 logging.info('RECEIVED %s', tt_status_report)
 
-                if self.__full_mode:
-                    ret = request_core.update_request_state(tt_status_report, session=session, logger=logger)
-                    METRICS.counter('update_request_state.{updated}').labels(updated=ret).inc()
-                else:
-                    try:
-                        logging.debug("Update request %s update time" % request_id)
-                        set_transfer_update_time(external_host, tt_status_report.external_id, datetime.datetime.utcnow() - datetime.timedelta(hours=24), session=session)
-                        METRICS.counter('set_transfer_update_time').inc()
-                    except Exception as error:
-                        logging.debug("Failed to update transfer's update time: %s" % str(error))
+                ret = request_core.update_request_state(tt_status_report, session=session, logger=logger)
+                METRICS.counter('update_request_state.{updated}').labels(updated=ret).inc()
         except Exception:
             logging.critical(traceback.format_exc())
 
 
-def receiver(id_, total_threads=1, full_mode=False, all_vos=False):
+def receiver(id_, total_threads=1, all_vos=False):
     """
     Main loop to consume messages from the FTS3 producer.
     """
 
-    logging.info('receiver starting in full mode: %s' % full_mode)
+    logging.info('receiver starting')
 
     logger_prefix = executable = 'conveyor-receiver'
 
@@ -171,8 +160,7 @@ def receiver(id_, total_threads=1, full_mode=False, all_vos=False):
                     METRICS.counter('reconnect.{host}').labels(host=conn.transport._Transport__host_and_ports[0][0].split('.')[0]).inc()
 
                     conn.set_listener('rucio-messaging-fts3', Receiver(broker=conn.transport._Transport__host_and_ports[0],
-                                                                       id_=id_, total_threads=total_threads,
-                                                                       full_mode=full_mode, all_vos=all_vos))
+                                                                       id_=id_, total_threads=total_threads, all_vos=all_vos))
                     if not use_ssl:
                         conn.connect(username, password, wait=True)
                     else:
@@ -197,7 +185,7 @@ def stop(signum=None, frame=None):
     graceful_stop.set()
 
 
-def run(once=False, total_threads=1, full_mode=False):
+def run(once=False, total_threads=1):
     """
     Starts up the receiver thread
     """
@@ -208,7 +196,6 @@ def run(once=False, total_threads=1, full_mode=False):
 
     logging.info('starting receiver thread')
     threads = [threading.Thread(target=receiver, kwargs={'id_': i,
-                                                         'full_mode': full_mode,
                                                          'total_threads': total_threads}) for i in range(0, total_threads)]
 
     [thread.start() for thread in threads]


Test patch:
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -429,7 +429,7 @@ def test_multisource_receiver(vo, did_factory, replica_client, root_account, met
     Run receiver as a background thread to automatically handle fts notifications.
     Ensure that a multi-source job in which the first source fails is correctly handled by receiver.
     """
-    receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1})
+    receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'all_vos': True, 'total_threads': 1})
     receiver_thread.start()
 
     try:
@@ -507,7 +507,7 @@ def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_acco
     """
     Verify that the receiver correctly handles multihop jobs which fail
     """
-    receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1})
+    receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'all_vos': True, 'total_threads': 1})
     receiver_thread.start()
 
     try:
@@ -565,7 +565,7 @@ def test_multihop_receiver_on_success(vo, did_factory, root_account, core_config
     """
     Verify that the receiver correctly handles successful multihop jobs
     """
-    receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1})
+    receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'all_vos': True, 'total_threads': 1})
     receiver_thread.start()
 
     try:
@@ -631,7 +631,7 @@ def _perform_request_update(self, msg, *, session=None, logger=logging.log):
             return ret
 
     with patch('rucio.daemons.conveyor.receiver.Receiver', ReceiverWrapper):
-        receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1})
+        receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'all_vos': True, 'total_threads': 1})
         receiver_thread.start()
         # Fake that destination RSE is a tape
         rse_core.update_rse(rse_id=dst_rse_id, parameters={'rse_type': RSEType.TAPE})