Repository: rucio/rucio

Issue:
fts-source-strategy configurable by activity
Motivation
----------
The conveyor supports `--fts-source-strategy` option to specify which option should be chosen for the sources. This option is the same for all the transfers processed by a given conveyor instance. It would be better to be able to specify different options for different activities.


Modification
------------
Introduce a default `default-fts-source-strategy` in the configuration table as well a `activity-fts-source-strategy`, so that every activities can use a different strategy.




Discussion:
I wonder if we should make this configurable via the config table?
> I wonder if we should make this configurable via the config table?

We should. IMHO all configuration that is not absolutely necessary to make a connection to the rucio server (client side), and to make the database connection (daemon, server side), should be moved into the config tables. New configs should not go into the .cfg
The `--fts-source-strategy` makes sense as an override, but per default I think it just should come from the config table. But re-reading the ticket, I guess that's what @cserf meant anyway.
I was thinking about completely getting rid of the `--fts-source-strategy` and just use the config table. If one has 2 ways to specify the source strategy, that's not obivious which one has the precedence on the other.
It might make sense to have both ways, if we just want to test a strategy at one conveyor, without impacting all of them. (Thus command line has precedence) - But I am not really sure if it is needed for the source selection strategy.

Resolving patch:
diff --git a/lib/rucio/daemons/conveyor/common.py b/lib/rucio/daemons/conveyor/common.py
--- a/lib/rucio/daemons/conveyor/common.py
+++ b/lib/rucio/daemons/conveyor/common.py
@@ -14,7 +14,7 @@
 #
 # Authors:
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2018
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2019
 # - Vincent Garonne <vgaronne@gmail.com>, 2014-2016
 # - Martin Barisits <martin.barisits@cern.ch>, 2014-2017
 # - Wen Guan <wguan.icedew@gmail.com>, 2014-2016
@@ -31,21 +31,23 @@
 """
 
 from __future__ import division
+from json import loads
 
 import datetime
 import logging
 import time
 import traceback
 
-from rucio.common.exception import InvalidRSEExpression, TransferToolTimeout, TransferToolWrongAnswer, RequestNotFound
+from rucio.common.config import config_get
+from rucio.common.exception import InvalidRSEExpression, TransferToolTimeout, TransferToolWrongAnswer, RequestNotFound, ConfigNotFound
 from rucio.common.utils import chunks
 from rucio.core import request, transfer as transfer_core
+from rucio.core.config import get
 from rucio.core.monitor import record_counter, record_timer
 from rucio.core.rse import list_rses
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.db.sqla.constants import RequestState
 from rucio.rse import rsemanager as rsemgr
-from rucio.common.config import config_get
 
 USER_ACTIVITY = config_get('conveyor', 'user_activities', False, ['user', 'user_test'])
 USER_TRANSFERS = config_get('conveyor', 'user_transfers', False, None)
@@ -162,14 +164,14 @@ def submit_transfer(external_host, job, submitter='submitter', logging_prepend_s
             logging.error(prepend_str + 'Failed to cancel transfers %s on %s with error: %s' % (eid, external_host, traceback.format_exc()))
 
 
-def bulk_group_transfer(transfers, policy='rule', group_bulk=200, fts_source_strategy='auto', max_time_in_queue=None):
+def bulk_group_transfer(transfers, policy='rule', group_bulk=200, source_strategy=None, max_time_in_queue=None):
     """
     Group transfers in bulk based on certain criterias
 
     :param transfers:             List of transfers to group.
     :param plicy:                 Policy to use to group.
     :param group_bulk:            Bulk sizes.
-    :param fts_source_strategy:   Strategy to group fts sources
+    :param source_strategy:       Strategy to group sources
     :param max_time_in_queue:     Maximum time in queue
     :return:                      List of grouped transfers.
     """
@@ -177,6 +179,20 @@ def bulk_group_transfer(transfers, policy='rule', group_bulk=200, fts_source_str
     grouped_transfers = {}
     grouped_jobs = {}
 
+    try:
+        default_source_strategy = get(section='conveyor', option='default-source-strategy')
+    except ConfigNotFound:
+        default_source_strategy = 'orderly'
+
+    try:
+        activity_source_strategy = get(section='conveyor', option='activity-source-strategy')
+        activity_source_strategy = loads(activity_source_strategy)
+    except ConfigNotFound:
+        activity_source_strategy = {}
+    except ValueError:
+        logging.warning('activity_source_strategy not properly defined')
+        activity_source_strategy = {}
+
     for request_id in transfers:
         transfer = transfers[request_id]
 
@@ -187,7 +203,7 @@ def bulk_group_transfer(transfers, policy='rule', group_bulk=200, fts_source_str
                 'filesize': int(transfer['file_metadata']['filesize']),
                 'checksum': None,
                 'verify_checksum': verify_checksum,
-                'selection_strategy': fts_source_strategy,
+                'selection_strategy': source_strategy if source_strategy else activity_source_strategy.get(str(transfer['file_metadata']['activity']), default_source_strategy),
                 'request_type': transfer['file_metadata'].get('request_type', None),
                 'activity': str(transfer['file_metadata']['activity'])}
         if verify_checksum != 'none':
diff --git a/lib/rucio/daemons/conveyor/stager.py b/lib/rucio/daemons/conveyor/stager.py
--- a/lib/rucio/daemons/conveyor/stager.py
+++ b/lib/rucio/daemons/conveyor/stager.py
@@ -17,7 +17,7 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2015-2017
 # - Vincent Garonne <vgaronne@gmail.com>, 2016-2018
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2017
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2018-2019
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 #
 # PY3K COMPATIBLE
@@ -61,7 +61,7 @@
 
 
 def stager(once=False, rses=None, mock=False, bulk=100, group_bulk=1, group_policy='rule',
-           fts_source_strategy='auto', activities=None, sleep_time=600, retry_other_fts=False):
+           source_strategy=None, activities=None, sleep_time=600, retry_other_fts=False):
     """
     Main loop to submit a new transfer primitive to a transfertool.
     """
@@ -147,7 +147,7 @@ def stager(once=False, rses=None, mock=False, bulk=100, group_bulk=1, group_poli
                 # group transfers
                 logging.info(prepend_str + 'Starting to group transfers for %s' % (activity))
                 start_time = time.time()
-                grouped_jobs = bulk_group_transfer(transfers, group_policy, group_bulk, fts_source_strategy, max_time_in_queue)
+                grouped_jobs = bulk_group_transfer(transfers, group_policy, group_bulk, source_strategy, max_time_in_queue)
                 record_timer('daemons.conveyor.stager.bulk_group_transfer', (time.time() - start_time) * 1000 / (len(transfers) if transfers else 1))
 
                 logging.info(prepend_str + 'Starting to submit transfers for %s' % (activity))
@@ -183,7 +183,7 @@ def stop(signum=None, frame=None):
 
 
 def run(once=False, total_threads=1, group_bulk=1, group_policy='rule',
-        mock=False, rses=None, include_rses=None, exclude_rses=None, bulk=100, fts_source_strategy='auto',
+        mock=False, rses=None, include_rses=None, exclude_rses=None, bulk=100, source_strategy=None,
         activities=[], sleep_time=600, retry_other_fts=False):
     """
     Starts up the conveyer threads.
@@ -209,7 +209,7 @@ def run(once=False, total_threads=1, group_bulk=1, group_policy='rule',
                bulk=bulk,
                group_bulk=group_bulk,
                group_policy=group_policy,
-               fts_source_strategy=fts_source_strategy,
+               source_strategy=source_strategy,
                activities=activities,
                retry_other_fts=retry_other_fts)
 
@@ -222,7 +222,7 @@ def run(once=False, total_threads=1, group_bulk=1, group_policy='rule',
                                                            'activities': activities,
                                                            'mock': mock,
                                                            'sleep_time': sleep_time,
-                                                           'fts_source_strategy': fts_source_strategy,
+                                                           'source_strategy': source_strategy,
                                                            'retry_other_fts': retry_other_fts}) for _ in range(0, total_threads)]
 
         [thread.start() for thread in threads]
diff --git a/lib/rucio/daemons/conveyor/submitter.py b/lib/rucio/daemons/conveyor/submitter.py
--- a/lib/rucio/daemons/conveyor/submitter.py
+++ b/lib/rucio/daemons/conveyor/submitter.py
@@ -66,7 +66,7 @@
 
 
 def submitter(once=False, rses=None, mock=False,
-              bulk=100, group_bulk=1, group_policy='rule', fts_source_strategy='auto',
+              bulk=100, group_bulk=1, group_policy='rule', source_strategy=None,
               activities=None, sleep_time=600, max_sources=4, retry_other_fts=False):
     """
     Main loop to submit a new transfer primitive to a transfertool.
@@ -169,7 +169,7 @@ def submitter(once=False, rses=None, mock=False,
                 logging.info(prepend_str + 'Starting to group transfers for %s' % (activity))
                 start_time = time.time()
 
-                grouped_jobs = bulk_group_transfer(transfers, group_policy, group_bulk, fts_source_strategy, max_time_in_queue)
+                grouped_jobs = bulk_group_transfer(transfers, group_policy, group_bulk, source_strategy, max_time_in_queue)
                 record_timer('daemons.conveyor.transfer_submitter.bulk_group_transfer', (time.time() - start_time) * 1000 / (len(transfers) if transfers else 1))
 
                 logging.info(prepend_str + 'Starting to submit transfers for %s' % (activity))
@@ -213,7 +213,7 @@ def stop(signum=None, frame=None):
 
 
 def run(once=False, group_bulk=1, group_policy='rule',
-        mock=False, rses=None, include_rses=None, exclude_rses=None, bulk=100, fts_source_strategy='auto',
+        mock=False, rses=None, include_rses=None, exclude_rses=None, bulk=100, source_strategy=None,
         activities=None, sleep_time=600, max_sources=4, retry_other_fts=False, total_threads=1):
     """
     Starts up the conveyer threads.
@@ -241,7 +241,7 @@ def run(once=False, group_bulk=1, group_policy='rule',
                                                           'mock': mock,
                                                           'sleep_time': sleep_time,
                                                           'max_sources': max_sources,
-                                                          'fts_source_strategy': fts_source_strategy,
+                                                          'source_strategy': source_strategy,
                                                           'retry_other_fts': retry_other_fts}) for _ in range(0, total_threads)]
 
     [thread.start() for thread in threads]


Test patch: