Repository: rucio/rucio

Issue:
receiver marks ARCHIVING requests as done
Motivation
----------

The receiver waits for a "Completion" message from FTS. And there is a very simple condition to transition the transfer to a "DONE" state on reception of such message: 
https://github.com/rucio/rucio/blob/b10d485fd31555d17ac1e4143380a3b09ab83a4a/lib/rucio/transfertool/fts3.py#L500

I tested in the dev containers and FTS, indeed, sends such a message before transitioning the transfer to "ARCHIVING" state. This is not the desired behavior, we don't want to mark ARCHIVING requests as DONE.

```
DEBUG   Fri, 06 May 2022 14:58:47 +0000; CO {
	"tr_id" : "2022-05-06-1458__xrd3__xrd4__6658118__04a8c30e-cd4d-11ec-b377-0242ac15000e",
	"endpnt" : "rucio/fts",
	"src_srm_v" : "",
	"dest_srm_v" : "",
	"vo" : "RucioUser@",
	"src_url" : "root://xrd3:1096//rucio/mock/9e/75/lfn_a992c42029c547e98ccddf0535a20c40_0",
	"dst_url" : "root://xrd4:1097//rucio/mock/9e/75/lfn_a992c42029c547e98ccddf0535a20c40_0",
	"src_hostname" : "xrd3",
	"dst_hostname" : "xrd4",
	"src_site_name" : "",
	"dst_site_name" : "",
	"t_channel" : "xrd3__xrd4",
	"timestamp_tr_st" : 1651849124437,
	"timestamp_tr_comp" : 1651849127143,
	"timestamp_chk_src_st" : 0,
	"timestamp_chk_src_ended" : 0,
	"timestamp_checksum_dest_st" : 0,
	"timestamp_checksum_dest_ended" : 0,
	"t_timeout" : 601,
	"chk_timeout" : 0,
	"t_error_code" : 0,
	"tr_error_scope" : "",
	"t_failure_phase" : "",
	"tr_error_category" : "",
	"t_final_transfer_state" : "Ok",
	"tr_bt_transfered" : 2,
	"nstreams" : 0,
	"buf_size" : 0,
	"tcp_buf_size" : 0,
	"block_size" : 0,
	"f_size" : 2,
	"time_srm_prep_st" : 0,
	"time_srm_prep_end" : 0,
	"time_srm_fin_st" : 0,
	"time_srm_fin_end" : 0,
	"srm_space_token_src" : "",
	"srm_space_token_dst" : "",
	"t__error_message" : "",
	"tr_timestamp_start" : 1651849122592,
	"tr_timestamp_complete" : 1651849127278,
	"channel_type" : "urlcopy",
	"user_dn" : "",
	"file_metadata" : {
		"dst_type" : "TAPE",
		"name" : "lfn_a992c42029c547e98ccddf0535a20c40_0",
		"adler32" : "010700b7",
		"src_rse" : "XRD3",
		"activity" : "User Subscriptions",
		"src_rse_id" : "257e34c526ff40389a5b414cad18fdb5",
		"request_id" : "599341fc69cc4ccc95cd62ea329f0d9b",
		"request_type" : "TRANSFER",
		"filesize" : 2,
		"dest_rse_id" : "92083ab3073c4d7fbe8defbdfc83c961",
		"scope" : "mock",
		"dst_rse" : "XRD4",
		"src_type" : "DISK",
		"md5" : "9ad3285ba2ed40f7b805800f927766fc"
	},
	"job_metadata" : {
		"auth_method" : "certificate",
		"multi_sources" : false,
		"issuer" : "rucio"
	},
	"retry" : 0,
	"retry_max" : 0,
	"job_m_replica" : false,
	"job_state" : "UNKNOWN",
	"is_recoverable" : false,
	"ipv6" : false,
	"final_destination" : "",
	"transfer_type" : "3rd pull"
}
INFO    Fri, 06 May 2022 14:58:47 +0000; Completion message: 2022-05-06-1458__xrd3__xrd4__6658118__04a8c30e-cd4d-11ec-b377-0242ac15000e
DEBUG   Fri, 06 May 2022 14:58:47 +0000; SS {
	"user_dn" : "",
	"src_url" : "root://xrd3:1096//rucio/mock/9e/75/lfn_a992c42029c547e98ccddf0535a20c40_0",
	"dst_url" : "root://xrd4:1097//rucio/mock/9e/75/lfn_a992c42029c547e98ccddf0535a20c40_0",
	"vo_name" : "RucioUser@",
	"source_se" : "root://xrd3",
	"dest_se" : "root://xrd4",
	"job_id" : "04a8c30e-cd4d-11ec-b377-0242ac15000e",
	"file_id" : 6658118,
	"job_state" : "ARCHIVING",
	"file_state" : "ARCHIVING",
	"retry_counter" : 0,
	"retry_max" : 0,
	"user_filesize" : 2,
	"timestamp" : 1651849127528,
	"staging" : false,
	"staging_start" : 0,
	"staging_finished" : 0,
	"submit_time" : 1651849122000,
	"reason" : "",
	"job_metadata" : {
		"auth_method" : "certificate",
		"multi_sources" : false,
		"issuer" : "rucio"
	},
	"file_metadata" : {
		"dst_type" : "TAPE",
		"name" : "lfn_a992c42029c547e98ccddf0535a20c40_0",
		"adler32" : "010700b7",
		"src_rse" : "XRD3",
		"activity" : "User Subscriptions",
		"src_rse_id" : "257e34c526ff40389a5b414cad18fdb5",
		"request_id" : "599341fc69cc4ccc95cd62ea329f0d9b",
		"request_type" : "TRANSFER",
		"filesize" : 2,
		"dest_rse_id" : "92083ab3073c4d7fbe8defbdfc83c961",
		"scope" : "mock",
		"dst_rse" : "XRD4",
		"src_type" : "DISK",
		"md5" : "9ad3285ba2ed40f7b805800f927766fc"
	}
}
INFO    Fri, 06 May 2022 14:58:47 +0000; State change: ARCHIVING 04a8c30e-cd4d-11ec-b377-0242ac15000e/6658118
```



Discussion:
FTS devs will add an `"archiving": true/false` field to Completion messages. Waiting for their implementation
Hello,

I will track this issue here: https://its.cern.ch/jira/browse/FTS-1451

Resolving patch:
diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py
--- a/lib/rucio/transfertool/fts3.py
+++ b/lib/rucio/transfertool/fts3.py
@@ -583,7 +583,7 @@ def initialize(self, session, logger=logging.log):
         reason = fts_message.get('t__error_message', None)
         # job_state = fts_message.get('t_final_transfer_state', None)
         new_state = None
-        if str(fts_message['t_final_transfer_state']) == FTS_COMPLETE_STATE.OK:  # pylint:disable=no-member
+        if str(fts_message['t_final_transfer_state']) == FTS_COMPLETE_STATE.OK and not fts_message.get('is_archiving'):  # pylint:disable=no-member
             new_state = RequestState.DONE
         elif str(fts_message['t_final_transfer_state']) == FTS_COMPLETE_STATE.ERROR:
             request = self.request(session)


Test patch:
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import logging
 import threading
 import time
 from datetime import datetime, timedelta
@@ -41,7 +41,7 @@
 from rucio.daemons.conveyor.submitter import submitter
 from rucio.daemons.conveyor.stager import stager
 from rucio.daemons.conveyor.throttler import throttler
-from rucio.daemons.conveyor.receiver import receiver, graceful_stop as receiver_graceful_stop
+from rucio.daemons.conveyor.receiver import receiver, graceful_stop as receiver_graceful_stop, Receiver
 from rucio.daemons.reaper.reaper import reaper
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import LockState, RequestState, RequestType, ReplicaState, RSEType, RuleState
@@ -599,6 +599,69 @@ def test_multihop_receiver_on_success(vo, did_factory, root_account, core_config
         receiver_graceful_stop.clear()
 
 
+@skip_rse_tests_with_accounts
+@pytest.mark.dirty(reason="leaves files in XRD containers")
+@pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter and receiver")
+def test_receiver_archiving(vo, did_factory, replica_client, root_account):
+    """
+    Ensure that receiver doesn't mark archiving requests as DONE
+    """
+
+    src_rse = 'XRD3'
+    src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
+    dst_rse = 'XRD4'
+    dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
+    all_rses = [src_rse_id, dst_rse_id]
+
+    received_messages = {}
+
+    class ReceiverWrapper(Receiver):
+        """
+        Wrap receiver to record the last handled message for each given request_id
+        """
+        def _perform_request_update(self, msg, *, session=None, logger=logging.log):
+            ret = super()._perform_request_update(msg, session=session, logger=logger)
+            received_messages[msg['file_metadata']['request_id']] = msg
+            return ret
+
+    with patch('rucio.daemons.conveyor.receiver.Receiver', ReceiverWrapper):
+        receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1})
+        receiver_thread.start()
+        # Fake that destination RSE is a tape
+        rse_core.update_rse(rse_id=dst_rse_id, parameters={'rse_type': RSEType.TAPE})
+        try:
+            rse_core.add_rse_attribute(dst_rse_id, 'archive_timeout', 60)
+
+            did = did_factory.upload_test_file(src_rse)
+            rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+            submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
+
+            # Wait for the reception of the FTS Completion message for the submitted request
+            request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+            for i in range(MAX_POLL_WAIT_SECONDS):
+                if request['id'] in received_messages:
+                    break
+                if i == MAX_POLL_WAIT_SECONDS - 1:
+                    assert False  # Waited too long; fail the test
+                time.sleep(1)
+            assert __wait_for_fts_state(request, expected_state='ARCHIVING') == 'ARCHIVING'
+
+            # Receiver must not mark "ARCHIVING" requests as "DONE"
+            request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+            assert request['state'] == RequestState.SUBMITTED
+            # Poller should also correctly handle "ARCHIVING" transfers and not mark them as DONE
+            poller(once=True, older_than=0, partition_wait_time=0)
+            request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+            assert request['state'] == RequestState.SUBMITTED
+        finally:
+            rse_core.update_rse(rse_id=dst_rse_id, parameters={'rse_type': RSEType.DISK})
+            rse_core.del_rse_attribute(dst_rse_id, 'archive_timeout')
+
+            receiver_graceful_stop.set()
+            receiver_thread.join(timeout=5)
+            receiver_graceful_stop.clear()
+
+
 @skip_rse_tests_with_accounts
 @pytest.mark.noparallel(reason="runs multiple conveyor daemons")
 @pytest.mark.parametrize("file_config_mock", [{