Repository: rucio/rucio
Issue: race condition between poller and cleaner

Motivation
----------
```
2022-04-04 10:59:52,346 root    1       CRITICAL        conveyor-poller[7/29]: Exception
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/rucio/core/request.py", line 1503, in update_request_state
    set_request_state(request_id, session=session, **fields_to_update)
  File "/usr/local/lib/python3.6/site-packages/rucio/db/sqla/session.py", line 390, in new_funct
    result = function(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/rucio/core/request.py", line 702, in set_request_state
    if state in [RequestState.FAILED, RequestState.DONE, RequestState.LOST] and (request["external_id"] != external_id):
TypeError: 'NoneType' object is not subscriptable
```


Probably to be fixed on poller side

Changed files:
lib/rucio/core/request.py, lib/rucio/transfertool/fts3.py

diff --git a/lib/rucio/core/request.py b/lib/rucio/core/request.py
index 4bbadde6d..4a84f1456 100644
--- a/lib/rucio/core/request.py
+++ b/lib/rucio/core/request.py
@@ -696,6 +696,11 @@ def set_request_state(request_id, state, external_id=None, transferred_at=None,
             update_items['attributes'] = json.dumps(attributes)
 
         request = get_request(request_id, session=session)
+        if not request:
+            # The request was deleted in the meantime. Ignore it.
+            logger(logging.WARNING, "Request %s not found. Cannot set its state to %s", request_id, state)
+            return
+
         if state in [RequestState.FAILED, RequestState.DONE, RequestState.LOST] and (request["external_id"] != external_id):
             logger(logging.ERROR, "Request %s should not be updated to 'Failed' or 'Done' without external transfer_id" % request_id)
         else:
diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py
index 00a55cbc8..b16ff23d2 100644
--- a/lib/rucio/transfertool/fts3.py
+++ b/lib/rucio/transfertool/fts3.py
@@ -506,7 +506,10 @@ class FTS3CompletionMessageTransferStatusReport(Fts3TransferStatusReport):
         transfer_id = self._transfer_id
         if new_state:
             request = self.request(session)
-            if request['external_id'] == transfer_id and request['state'] != new_state:
+            if not request:
+                logger(logging.WARNING, '%s: no request with this id in the database. Skipping. external_id: %s (%s). new_state: %s', request_id, transfer_id, self.external_host, new_state)
+                return
+            if request and request['external_id'] == transfer_id and request['state'] != new_state:
                 src_rse_name, src_rse_id = self._find_used_source_rse(session, logger)
 
                 self._reason = reason
@@ -591,6 +594,9 @@ class FTS3ApiTransferStatusReport(Fts3TransferStatusReport):
         transfer_id = self._transfer_id
         if new_state:
             request = self.request(session)
+            if not request:
+                logger(logging.WARNING, '%s: no request with this id in the database. Skipping. external_id: %s (%s). new_state: %s', request_id, transfer_id, self.external_host, new_state)
+                return
             if request['external_id'] == transfer_id and request['state'] != new_state:
                 src_rse_name, src_rse_id = self._find_used_source_rse(session, logger)