Repository: rucio/rucio

Issue:
Implement, in rucio 1.27, forward compatibility with the 1.28 behavior
Motivation
----------
The 1.28 release brings some behavioral changes which don't allow running both rucio 1.27 and 1.28 instances on the same database
in parallel. As we don't want to perform a "big bang" update, and would like to run these two version at the same time for validation, some adaptations are needed. 

In particular, 1.28 brings  a new transfers_hop table. The transfers created by this version of rucio will fill that table, which has foreign key constraints pointing to the requests table. A rucio 1.27 finisher/server cannot correctly archive a request created by 1.28: request deletion will fail with foreign key error. 

Another problematic case is the usage of third_party_copy _read/_write in 1.28. The alembic migrations correctly fill these columns from third_party_copy. However, a 1.27 server running after the alembic migration will only fill "third_party_copy". Resulting in errors. 







Discussion:

Resolving patch:
diff --git a/lib/rucio/core/request.py b/lib/rucio/core/request.py
--- a/lib/rucio/core/request.py
+++ b/lib/rucio/core/request.py
@@ -44,11 +44,15 @@
 from itertools import filterfalse
 from typing import TYPE_CHECKING
 
+from dogpile.cache.api import NoValue
 from six import string_types
-from sqlalchemy import and_, or_, func, update
+from sqlalchemy import and_, or_, func, update, inspect
 from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.sql.expression import asc, true
+from sqlalchemy.schema import Table, Column
 
+from rucio.common.cache import make_region_memcached
 from rucio.common.config import config_get_bool, config_get
 from rucio.common.constants import FTS_STATE
 from rucio.common.exception import RequestNotFound, RucioException, UnsupportedOperation
@@ -60,7 +64,8 @@
 from rucio.core.rse import get_rse_name, get_rse_vo, get_rse_transfer_limits, get_rse_attribute
 from rucio.db.sqla import models, filter_thread_work
 from rucio.db.sqla.constants import RequestState, RequestType, LockState, RequestErrMsg
-from rucio.db.sqla.session import read_session, transactional_session, stream_session
+from rucio.db.sqla.session import read_session, transactional_session, stream_session, DEFAULT_SCHEMA_NAME
+from rucio.db.sqla.types import GUID
 from rucio.transfertool.fts3 import FTS3Transfertool
 
 RequestAndState = namedtuple('RequestAndState', ['request_id', 'request_state'])
@@ -80,6 +85,54 @@
 Requests accessed by external_id (So called transfers), are covered in the core transfer.py
 """
 
+REGION = make_region_memcached(expiration_time=120)
+TRANSFERS_HOP_TABLE = None
+
+
+def __delete_from_transfers_hops_table_if_it_exists(request_id, session, logger):
+    """
+    This function implements forward compatibility of the current 1.27 rucio release line
+    with the 1.28 rucio release which introduces a new table.
+    The goal is to allow parallel execution of 1.27 and 1.28 during the migration phase.
+    """
+    global TRANSFERS_HOP_TABLE
+
+    if not TRANSFERS_HOP_TABLE:
+        table_name = 'transfer_hops'
+
+        cache_key = 'hast_transfers_hop_table'
+        db_has_table = REGION.get(cache_key)
+        if isinstance(db_has_table, NoValue):
+            db_has_table = inspect(session.bind).has_table(table_name, schema=DEFAULT_SCHEMA_NAME)
+            REGION.set(cache_key, db_has_table)
+
+        if db_has_table:
+            base = declarative_base()
+            columns = [
+                Column('request_id', GUID()),
+                Column('next_hop_request_id', GUID()),
+                Column('initial_request_id', GUID()),
+            ]
+            table = Table(
+                table_name,
+                base.metadata,
+                *columns,
+                schema=DEFAULT_SCHEMA_NAME,
+            )
+
+            class DeclarativeObj(base):
+                __table__ = table
+                __mapper_args__ = {
+                    "primary_key": columns,
+                }
+            TRANSFERS_HOP_TABLE = DeclarativeObj
+
+    if TRANSFERS_HOP_TABLE:
+        logger(logging.DEBUG, "Deleting from transfer_hops table for request %s", request_id)
+        session.query(TRANSFERS_HOP_TABLE).filter(or_(TRANSFERS_HOP_TABLE.request_id == request_id,
+                                                      TRANSFERS_HOP_TABLE.next_hop_request_id == request_id,
+                                                      TRANSFERS_HOP_TABLE.initial_request_id == request_id)).delete()
+
 
 def should_retry_request(req, retry_protocol_mismatches):
     """
@@ -124,7 +177,7 @@ def requeue_and_archive(request, source_ranking_update=True, retry_protocol_mism
 
     if new_req:
         new_req['sources'] = get_sources(request_id, session=session)
-        archive_request(request_id, session=session)
+        archive_request(request_id, session=session, logger=logger)
 
         if should_retry_request(new_req, retry_protocol_mismatches):
             new_req['request_id'] = generate_uuid()
@@ -648,11 +701,12 @@ def get_request_by_did(scope, name, rse_id, request_type=None, session=None):
 
 
 @transactional_session
-def archive_request(request_id, session=None):
+def archive_request(request_id, session=None, logger=logging.log):
     """
     Move a request to the history table.
 
     :param request_id:  Request-ID as a 32 character hex string.
+    :param logger:      decorated logger instance
     :param session:     Database session to use.
     """
 
@@ -697,6 +751,7 @@ def archive_request(request_id, session=None):
             time_diff_s = time_diff.seconds + time_diff.days * 24 * 3600
             record_timer('core.request.archive_request.{activity}', time_diff_s, labels={'activity': req['activity'].replace(' ', '_')})
             session.query(models.Source).filter_by(request_id=request_id).delete()
+            __delete_from_transfers_hops_table_if_it_exists(request_id=request_id, session=session, logger=logger)
             session.query(models.Request).filter_by(id=request_id).delete()
         except IntegrityError as error:
             raise RucioException(error.args)
@@ -740,7 +795,7 @@ def cancel_request_did(scope, name, dest_rse_id, request_type=RequestType.TRANSF
                 transfertool_map[req[2]].cancel(transfer_ids=[req[1]])
             except Exception as error:
                 logger(logging.WARNING, 'Could not cancel FTS3 transfer %s on %s: %s' % (req[1], req[2], str(error)))
-        archive_request(request_id=req[0], session=session)
+        archive_request(request_id=req[0], session=session, logger=logger)
 
 
 def cancel_request_external_id(transfertool_obj, transfer_id):
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -925,6 +925,11 @@ def add_protocol(rse_id, parameter, session=None):
                 parameter[op_name] = parameter['domains'][s][op]
         del parameter['domains']
 
+    # This is needed for forward-compatibility with the 1.28 version, which relies on _read/_write columns
+    if parameter.get('third_party_copy') is not None and parameter.get('third_party_copy_read') is None and parameter.get('third_party_copy_write') is None:
+        parameter['third_party_copy_read'] = parameter['third_party_copy']
+        parameter['third_party_copy_write'] = parameter['third_party_copy']
+
     if ('extended_attributes' in parameter) and parameter['extended_attributes']:
         try:
             parameter['extended_attributes'] = json.dumps(parameter['extended_attributes'], separators=(',', ':'))
diff --git a/lib/rucio/daemons/conveyor/finisher.py b/lib/rucio/daemons/conveyor/finisher.py
--- a/lib/rucio/daemons/conveyor/finisher.py
+++ b/lib/rucio/daemons/conveyor/finisher.py
@@ -408,7 +408,7 @@ def __update_bulk_replicas(replicas, session=None, logger=logging.log):
 
     for replica in replicas:
         if not replica['archived']:
-            request_core.archive_request(replica['request_id'], session=session)
+            request_core.archive_request(replica['request_id'], session=session, logger=logger)
         logger(logging.INFO, "HANDLED REQUEST %s DID %s:%s AT RSE %s STATE %s", replica['request_id'], replica['scope'], replica['name'], replica['rse_id'], str(replica['state']))
     return True
 
@@ -427,7 +427,7 @@ def __update_replica(replica, session=None, logger=logging.log):
     try:
         replica_core.update_replicas_states([replica], nowait=True, session=session)
         if not replica['archived']:
-            request_core.archive_request(replica['request_id'], session=session)
+            request_core.archive_request(replica['request_id'], session=session, logger=logger)
         logger(logging.INFO, "HANDLED REQUEST %s DID %s:%s AT RSE %s STATE %s", replica['request_id'], replica['scope'], replica['name'], replica['rse_id'], str(replica['state']))
     except (UnsupportedOperation, ReplicaNotFound) as error:
         logger(logging.WARNING, "ERROR WHEN HANDLING REQUEST %s DID %s:%s AT RSE %s STATE %s: %s", replica['request_id'], replica['scope'], replica['name'], replica['rse_id'], str(replica['state']), str(error))
@@ -445,7 +445,7 @@ def __update_replica(replica, session=None, logger=logging.log):
                                          tombstone=datetime.datetime.utcnow(),
                                          session=session)
             if not replica['archived']:
-                request_core.archive_request(replica['request_id'], session=session)
+                request_core.archive_request(replica['request_id'], session=session, logger=logger)
             logger(logging.INFO, "HANDLED REQUEST %s DID %s:%s AT RSE %s STATE %s", replica['request_id'], replica['scope'], replica['name'], replica['rse_id'], str(replica['state']))
         except Exception as error:
             logger(logging.ERROR, 'Cannot register replica for DID %s:%s at RSE %s - potential dark data - %s', replica['scope'], replica['name'], replica['rse_id'], str(error))


Test patch: