Repository: rucio/rucio

Issue:
Create a prototype workflow with temporary tables
Motivation
----------
For bulk queries, instead of making the complicated OR connected SELECT queries, create a prototype workflow using temporary tables (Insert to the temp table, then join)

Modification
------------
1. Pick a suitable workflow to try this
2. Implement prototype as an optional workflow
3. Test




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -57,7 +57,7 @@
 from sqlalchemy.sql.expression import false
 
 from rucio.common import constants
-from rucio.common.config import config_get
+from rucio.common.config import config_get, config_get_bool
 from rucio.common.constants import SUPPORTED_PROTOCOLS, FTS_STATE
 from rucio.common.exception import (InvalidRSEExpression, NoDistance,
                                     RequestNotFound, RSEProtocolNotSupported,
@@ -75,6 +75,7 @@
 from rucio.db.sqla import models, filter_thread_work
 from rucio.db.sqla.constants import DIDType, RequestState, RSEType, RequestType, ReplicaState
 from rucio.db.sqla.session import read_session, transactional_session
+from rucio.db.sqla.util import create_temp_table
 from rucio.rse import rsemanager as rsemgr
 from rucio.transfertool.transfertool import Transfertool, TransferToolBuilder
 from rucio.transfertool.fts3 import FTS3Transfertool
@@ -1652,6 +1653,18 @@ def __list_transfer_requests_and_source_replicas(
     else:
         sub_requests = sub_requests.with_hint(models.Request, "INDEX(REQUESTS REQUESTS_TYP_STA_UPD_IDX)", 'oracle')
 
+    use_temp_tables = config_get_bool('core', 'use_temp_tables', default=False)
+    if rses and use_temp_tables:
+        temp_table_cls = create_temp_table(
+            "list_transfer_requests_and_source_replicas",
+            models.Column("rse_id", models.GUID()),
+            session=session,
+        )
+
+        session.bulk_insert_mappings(temp_table_cls, [{'rse_id': rse_id} for rse_id in rses])
+
+        sub_requests = sub_requests.join(temp_table_cls, temp_table_cls.rse_id == models.RSE.id)
+
     sub_requests = filter_thread_work(session=session, query=sub_requests, total_threads=total_workers, thread_id=worker_number, hash_variable=partition_hash_var)
 
     if limit:
@@ -1707,8 +1720,8 @@ def __list_transfer_requests_and_source_replicas(
     for (request_id, rule_id, scope, name, md5, adler32, byte_count, activity, attributes, previous_attempt_id, dest_rse_id, account, retry_count,
          priority, transfertool, source_rse_id, source_rse_name, file_path, source_ranking, source_url, distance_ranking) in query:
 
-        # rses (of unknown length) should be a temporary table to check against instead of this special case
-        if rses and dest_rse_id not in rses:
+        # If we didn't pre-filter using temporary tables on database side, perform the filtering here
+        if not use_temp_tables and rses and dest_rse_id not in rses:
             continue
 
         request = requests_by_id.get(request_id)
diff --git a/lib/rucio/db/sqla/util.py b/lib/rucio/db/sqla/util.py
--- a/lib/rucio/db/sqla/util.py
+++ b/lib/rucio/db/sqla/util.py
@@ -38,7 +38,8 @@
 from sqlalchemy import func, inspect
 from sqlalchemy.dialects.postgresql.base import PGInspector
 from sqlalchemy.exc import IntegrityError, DatabaseError
-from sqlalchemy.schema import CreateSchema, MetaData, Table, DropTable, ForeignKeyConstraint, DropConstraint
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.schema import CreateSchema, MetaData, Table, CreateTable, DropTable, ForeignKeyConstraint, DropConstraint
 from sqlalchemy.sql.ddl import DropSchema
 from sqlalchemy.sql.expression import select, text
 
@@ -308,3 +309,59 @@ def try_drop_constraint(constraint_name, table_name):
         op.drop_constraint(constraint_name, table_name)
     except DatabaseError as e:
         assert 'nonexistent constraint' in str(e)
+
+
+def create_temp_table(name, *columns, primary_key=None, session=None):
+    """
+    Create a temporary table with the given columns, register it into a declarative base, and return it.
+
+    Declarative definition _requires_ a primary key. It should be a subset of '*columns' argument
+    (either a single column, or a list). If not explicitly give, will use the first column as primary key.
+    This primary key is "fake", because it only exists in sqlalchemy and not in the database.
+
+    Mysql and sqlite don't support automatic cleanup of temporary tables on commit. This means that a
+    temporary table definition is preserved for the lifetime of a session. A session is regularly
+    re-used by sqlalchemy, that's why we have to assume the required temporary table already exist and
+    could contain data from a previous transaction. Drop all data from that table.
+    """
+    if session.bind.dialect.name == 'oracle':
+        additional_kwargs = {
+            'oracle_on_commit': 'DROP DEFINITION',
+            'prefixes': ['PRIVATE TEMPORARY'],
+        }
+        # PRIVATE_TEMP_TABLE_PREFIX, which defaults to "ORA$PTT_", _must_ prefix the name
+        name = f"ORA$PTT_{name}"
+    elif session.bind.dialect.name == 'postgresql':
+        additional_kwargs = {
+            'postgresql_on_commit': 'DROP',
+            'prefixes': ['TEMPORARY'],
+        }
+    else:
+        additional_kwargs = {
+            'prefixes': ['TEMPORARY'],
+        }
+
+    base = declarative_base()
+    table = Table(
+        name,
+        base.metadata,
+        *columns,
+        schema=None,  # Temporary tables exist in a special schema, so a schema name cannot be given when creating a temporary table
+        **additional_kwargs,
+    )
+
+    class DeclarativeObj(base):
+        __table__ = table
+        # The declarative base requires a primary key, even if it doesn't exit in the database
+        __mapper_args__ = {
+            "primary_key": primary_key if primary_key else columns[0],
+        }
+
+    # Ensure the table exists and is empty. On some database types, it can contain leftover data from a previous transaction
+    # executed by sqlalchemy within the same session (which is being re-used now)
+    if session.bind.dialect.name in ('oracle', 'postgresql'):
+        session.execute(CreateTable(table))
+    else:
+        session.execute(CreateTable(table, if_not_exists=True))
+        session.query(DeclarativeObj).delete()
+    return DeclarativeObj


Test patch:
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -134,7 +134,20 @@ def __get_source(request_id, src_rse_id, scope, name, session=None):
     'rucio.core.config.REGION',
     'rucio.daemons.reaper.reaper.REGION',
 ]}], indirect=True)
-def test_multihop_intermediate_replica_lifecycle(vo, did_factory, root_account, core_config_mock, caches_mock, metrics_mock):
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {
+        "overrides": [
+            ('core', 'use_temp_tables', 'True'),
+        ]
+    },
+    {
+        "overrides": [
+            ('core', 'use_temp_tables', 'False'),
+        ]
+    }
+], indirect=True)
+def test_multihop_intermediate_replica_lifecycle(vo, did_factory, root_account, core_config_mock, caches_mock, metrics_mock, file_config_mock):
     """
     Ensure that intermediate replicas created by the submitter are protected from deletion even if their tombstone is
     set to epoch.
diff --git a/lib/rucio/tests/test_conveyor_submitter.py b/lib/rucio/tests/test_conveyor_submitter.py
--- a/lib/rucio/tests/test_conveyor_submitter.py
+++ b/lib/rucio/tests/test_conveyor_submitter.py
@@ -36,7 +36,20 @@
 
 
 @pytest.mark.noparallel(reason="multiple submitters cannot be run in parallel due to partial job assignment by hash")
-def test_request_submitted_in_order(rse_factory, did_factory, root_account):
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {
+        "overrides": [
+            ('core', 'use_temp_tables', 'True'),
+        ]
+    },
+    {
+        "overrides": [
+            ('core', 'use_temp_tables', 'False'),
+        ]
+    }
+], indirect=True)
+def test_request_submitted_in_order(rse_factory, did_factory, root_account, file_config_mock):
 
     src_rses = [rse_factory.make_posix_rse() for _ in range(2)]
     dst_rses = [rse_factory.make_posix_rse() for _ in range(3)]
@@ -94,9 +107,21 @@ def _forge_requests_creation_time(session=None):
 
 
 @pytest.mark.noparallel(reason="multiple submitters cannot be run in parallel due to partial job assignment by hash")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True)
-]}], indirect=True)
+@pytest.mark.parametrize("core_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {
+        "table_content": [
+            ('transfers', 'use_multihop', True),
+            ('core', 'use_temp_tables', False),
+        ]
+    },
+    {
+        "table_content": [
+            ('transfers', 'use_multihop', True),
+            ('core', 'use_temp_tables', True),
+        ]
+    }
+], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
     'rucio.core.config.REGION',