Repository: rucio/rucio
Issue: Conveyor crash within activity loop restarts activities

Motivation
----------
Currently, if the conveyor-submitter loop crashes within the activity loop on e.g. 
```
2020-07-06 13:31:05,450 24684   CRITICAL        Thread [7/30] :  Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/rucio/daemons/conveyor/submitter.py", line 170, in submitter
    retry_other_fts=retry_other_fts)
  File "/usr/lib/python2.7/site-packages/rucio/daemons/conveyor/submitter.py", line 324, in __get_transfers
    request_core.set_requests_state(reqs_no_source, RequestState.NO_SOURCES)
  File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 365, in new_funct
    result = function(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/rucio/core/request.py", line 526, in set_requests_state
    set_request_state(request_id, new_state, session=session)
  File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 381, in new_funct
    result = function(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/rucio/core/request.py", line 509, in set_request_state
    raise UnsupportedOperation("Request %s state cannot be updated." % request_id)
UnsupportedOperation: The resource doesn't support the requested operation.
Details: Request 8b8bbc20c21e4471a407b50573d39944 state cannot be updated.
```
the entire activity loop is restarted, starting at the first activity again and thus starving the others.

Modification
------------
The try/except block needs to cover for this and not restart the loop.


Changed files:
lib/rucio/daemons/conveyor/submitter.py, lib/rucio/core/transfer.py

diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
index dc2cb09f6..3552a8eef 100644
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -1123,7 +1123,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                     new_req = queue_requests(requests=[{'dest_rse_id': dest_rse_id,
                                                         'scope': scope,
                                                         'name': name,
-                                                        'rule_id': None,
+                                                        'rule_id': '00000000000000000000000000000000',  # Dummy Rule ID used for multihop. TODO: Replace with actual rule_id once we can flag intermediate requests
                                                         'attributes': req_attributes,
                                                         'request_type': RequestType.TRANSFER,
                                                         'retry_count': retry_count,
@@ -1215,6 +1215,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                     transfers[new_req_id] = {'request_id': new_req_id,
                                              'initial_request_id': req_id,
                                              'parent_request': parent_request,
+                                             'account': InternalAccount('root'),
                                              'schemes': __add_compatible_schemes(schemes=[destination_protocol], allowed_schemes=current_schemes),
                                              # 'src_urls': [source_url],
                                              'sources': [(source_rse_name, source_url, source_rse_id, 0, 0)],
diff --git a/lib/rucio/daemons/conveyor/submitter.py b/lib/rucio/daemons/conveyor/submitter.py
index 3c3ed4f5d..88845bc2b 100644
--- a/lib/rucio/daemons/conveyor/submitter.py
+++ b/lib/rucio/daemons/conveyor/submitter.py
@@ -17,7 +17,7 @@
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2019
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Vincent Garonne <vgaronne@gmail.com>, 2014-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2014-2017
+# - Martin Barisits <martin.barisits@cern.ch>, 2014-2020
 # - Wen Guan <wguan.icedew@gmail.com>, 2014-2016
 # - Tomas Kouba <tomas.kouba@cern.ch>, 2014
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2016
@@ -133,22 +133,21 @@ def submitter(once=False, rses=None, mock=False,
     logging.info('%s Transfer submitter started', prepend_str)
 
     while not graceful_stop.is_set():
-
-        try:
-            heart_beat = heartbeat.live(executable, hostname, pid, hb_thread, older_than=3600)
-            prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
-
-            if activities is None:
-                activities = [None]
-            if rses:
-                rse_ids = [rse['id'] for rse in rses]
-            else:
-                rse_ids = None
-            for activity in activities:
+        if activities is None:
+            activities = [None]
+        if rses:
+            rse_ids = [rse['id'] for rse in rses]
+        else:
+            rse_ids = None
+        for activity in activities:
+            try:
                 if activity_next_exe_time[activity] > time.time():
                     graceful_stop.wait(1)
                     continue
 
+                heart_beat = heartbeat.live(executable, hostname, pid, hb_thread, older_than=3600)
+                prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
+
                 user_transfer = False
 
                 if activity in USER_ACTIVITY and USER_TRANSFERS in ['cms']:
@@ -222,8 +221,8 @@ def submitter(once=False, rses=None, mock=False,
                     logging.info('%s Only %s transfers for %s which is less than group bulk %s, sleep %s seconds', prepend_str, len(transfers), activity, group_bulk, sleep_time)
                     if activity_next_exe_time[activity] < time.time():
                         activity_next_exe_time[activity] = time.time() + sleep_time
-        except Exception:
-            logging.critical('%s %s', prepend_str, str(traceback.format_exc()))
+            except Exception:
+                logging.critical('%s %s', prepend_str, str(traceback.format_exc()))
 
         if once:
             break