Repository: rucio/rucio

Issue:
submitter logs too verbose
Motivation
----------

The monitoring team notified us about taking too much space for storing our logs. Take actions to reduce 


Modification
------------

Reduce the amount of data sent  while keeping the usefullness of logs



Discussion:

Resolving patch:
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -1681,3 +1681,107 @@ def __call__(self, parser, namespace, values, option_string=None):
             # The logger gets typically initialized after the argument parser
             # to set the verbosity of the logger. Thus using simple print to console.
             print("Warning: The commandline argument {} is deprecated! Please use {} in the future.".format(option_string, self.new_option_string))
+
+
+class PriorityQueue:
+    """
+    Heap-based [1] priority queue which supports priority update operations
+
+    It is used as a dictionary: pq['element'] = priority
+    The element with the highest priority can be accessed with pq.top() or pq.pop(),
+    depending on the desire to keep it in the heap or not.
+
+    [1] https://en.wikipedia.org/wiki/Heap_(data_structure)
+    """
+    class ContainerSlot:
+        def __init__(self, position, priority):
+            self.pos = position
+            self.prio = priority
+
+    def __init__(self):
+        self.heap = []
+        self.container = {}
+        self.empty_slots = []
+
+    def __len__(self):
+        return len(self.heap)
+
+    def __getitem__(self, item):
+        return self.container[item].prio
+
+    def __setitem__(self, key, value):
+        if key in self.container:
+            existing_prio = self.container[key].prio
+            self.container[key].prio = value
+            if value < existing_prio:
+                self._priority_decreased(key)
+            elif existing_prio < value:
+                self._priority_increased(key)
+        else:
+            self.heap.append(key)
+            self.container[key] = self.ContainerSlot(position=len(self.heap) - 1, priority=value)
+            self._priority_decreased(key)
+
+    def __contains__(self, item):
+        return item in self.container
+
+    def top(self):
+        return self.heap[0]
+
+    def pop(self):
+        item = self.heap[0]
+        self.container.pop(item)
+
+        tmp_item = self.heap.pop()
+        if self.heap:
+            self.heap[0] = tmp_item
+            self.container[tmp_item].pos = 0
+            self._priority_increased(tmp_item)
+        return item
+
+    def _priority_decreased(self, item):
+        heap_changed = False
+
+        pos = self.container[item].pos
+        pos_parent = (pos - 1) // 2
+        while pos > 0 and self.container[self.heap[pos]].prio < self.container[self.heap[pos_parent]].prio:
+            tmp_item, parent = self.heap[pos], self.heap[pos_parent] = self.heap[pos_parent], self.heap[pos]
+            self.container[tmp_item].pos, self.container[parent].pos = self.container[parent].pos, self.container[tmp_item].pos
+
+            pos = pos_parent
+            pos_parent = (pos - 1) // 2
+
+            heap_changed = True
+        return heap_changed
+
+    def _priority_increased(self, item):
+        heap_changed = False
+        heap_len = len(self.heap)
+        pos = self.container[item].pos
+        pos_child1 = 2 * pos + 1
+        pos_child2 = 2 * pos + 2
+
+        heap_restored = False
+        while not heap_restored:
+            # find minimum between item, child1, and child2
+            if pos_child1 < heap_len and self.container[self.heap[pos_child1]].prio < self.container[self.heap[pos]].prio:
+                pos_min = pos_child1
+            else:
+                pos_min = pos
+            if pos_child2 < heap_len and self.container[self.heap[pos_child2]].prio < self.container[self.heap[pos_min]].prio:
+                pos_min = pos_child2
+
+            if pos_min != pos:
+                _, tmp_item = self.heap[pos_min], self.heap[pos] = self.heap[pos], self.heap[pos_min]
+                self.container[tmp_item].pos = pos
+
+                pos = pos_min
+                pos_child1 = 2 * pos + 1
+                pos_child2 = 2 * pos + 2
+
+                heap_changed = True
+            else:
+                heap_restored = True
+
+        self.container[self.heap[pos]].pos = pos
+        return heap_changed
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -548,11 +548,12 @@ def set_transfers_state(transfers, state, submitted_at, external_host, external_
     :param session:    Database session to use.
     """
 
-    logger(logging.DEBUG, 'Start register transfer state to %s for eid %s' % (state, external_id))
+    logger(logging.INFO, 'Setting state(%s), external_host(%s) and eid(%s) for transfers: %s',
+           state.name, external_host, external_id, ', '.join(t.rws.request_id for t in transfers))
     try:
         for transfer in transfers:
             rws = transfer.rws
-            logger(logging.INFO, 'COPYING REQUEST %s DID %s:%s USING %s with state(%s) with eid(%s)' % (rws.request_id, rws.scope, rws.name, external_host, state, external_id))
+            logger(logging.DEBUG, 'COPYING REQUEST %s DID %s:%s USING %s with state(%s) with eid(%s)' % (rws.request_id, rws.scope, rws.name, external_host, state, external_id))
             rowcount = session.query(models.Request)\
                               .filter_by(id=transfer.rws.request_id)\
                               .filter(models.Request.state == RequestState.SUBMITTING)\
diff --git a/lib/rucio/daemons/conveyor/submitter.py b/lib/rucio/daemons/conveyor/submitter.py
--- a/lib/rucio/daemons/conveyor/submitter.py
+++ b/lib/rucio/daemons/conveyor/submitter.py
@@ -51,6 +51,7 @@
 from rucio.common.config import config_get, config_get_bool
 from rucio.common.logging import setup_logging
 from rucio.common.schema import get_schema_value
+from rucio.common.utils import PriorityQueue
 from rucio.core import transfer as transfer_core
 from rucio.core.monitor import MultiCounter, record_timer
 from rucio.daemons.conveyor.common import submit_transfer, get_conveyor_rses, HeartbeatHandler
@@ -129,6 +130,13 @@ def submitter(once=False, rses=None, partition_wait_time=10,
     if filter_transfertool:
         executable += ' --filter-transfertool ' + filter_transfertool
 
+    if activities is None:
+        activities = [None]
+    if rses:
+        rse_ids = [rse['id'] for rse in rses]
+    else:
+        rse_ids = None
+
     with HeartbeatHandler(executable=executable, logger_prefix=logger_prefix) as heartbeat_handler:
         logger = heartbeat_handler.logger
         logger(logging.INFO, 'Submitter starting with timeout %s', timeout)
@@ -136,86 +144,90 @@ def submitter(once=False, rses=None, partition_wait_time=10,
         if partition_wait_time:
             graceful_stop.wait(partition_wait_time)
 
-        while not graceful_stop.is_set():
-            if activities is None:
-                activities = [None]
-            if rses:
-                rse_ids = [rse['id'] for rse in rses]
-            else:
-                rse_ids = None
-            for activity in activities:
-                try:
-                    if activity_next_exe_time[activity] > time.time():
-                        graceful_stop.wait(1)
-                        continue
-
-                    heart_beat, logger = heartbeat_handler.live(older_than=3600)
-
-                    logger(logging.INFO, 'Starting to get transfer transfers for %s', activity)
+        activity_next_exe_time = PriorityQueue()
+        for activity in activities:
+            activity_next_exe_time[activity] = time.time()
+
+        while not graceful_stop.is_set() and activity_next_exe_time:
+            try:
+                time_to_sleep = 0
+                if once:
+                    activity = activity_next_exe_time.pop()
+                else:
+                    activity = activity_next_exe_time.top()
+                    time_to_sleep = activity_next_exe_time[activity] - time.time()
+                    activity_next_exe_time[activity] = time.time() + 1
+                if time_to_sleep > 0:
+                    logger(logging.DEBUG, 'Switching to activity %s and sleeping %s seconds', activity, time_to_sleep)
+                    graceful_stop.wait(time_to_sleep)
+                else:
+                    logger(logging.DEBUG, 'Switching to activity %s', activity)
+
+                heart_beat, logger = heartbeat_handler.live(older_than=3600)
+
+                start_time = time.time()
+
+                transfertool_kwargs = {
+                    FTS3Transfertool: {
+                        'group_policy': group_policy,
+                        'group_bulk': group_bulk,
+                        'source_strategy': source_strategy,
+                        'max_time_in_queue': max_time_in_queue,
+                        'bring_online': bring_online,
+                        'default_lifetime': 172800,
+                        'archive_timeout_override': archive_timeout_override,
+                    },
+                    GlobusTransferTool: {
+                        'group_policy': transfertype,
+                        'group_bulk': group_bulk,
+                    },
+                }
+                transfers = transfer_core.next_transfers_to_submit(
+                    total_workers=heart_beat['nr_threads'],
+                    worker_number=heart_beat['assign_thread'],
+                    partition_hash_var=partition_hash_var,
+                    failover_schemes=failover_scheme,
+                    limit=bulk,
+                    activity=activity,
+                    rses=rse_ids,
+                    schemes=scheme,
+                    filter_transfertool=filter_transfertool,
+                    transfertools_by_name={transfertool: TRANSFERTOOL_CLASSES_BY_NAME[transfertool]},
+                    older_than=None,
+                    request_type=RequestType.TRANSFER,
+                    ignore_availability=ignore_availability,
+                    logger=logger,
+                )
+                total_transfers = len(list(hop for paths in transfers.values() for path in paths for hop in path))
+
+                record_timer('daemons.conveyor.transfer_submitter.get_transfers.per_transfer', (time.time() - start_time) * 1000 / (total_transfers or 1))
+                GET_TRANSFERS_COUNTER.inc(total_transfers)
+                record_timer('daemons.conveyor.transfer_submitter.get_transfers.transfers', total_transfers)
+                logger(logging.INFO, '%sGot %s transfers for %s in %s seconds',
+                       'Slept %s seconds, then ' % time_to_sleep if time_to_sleep > 0 else '',
+                       total_transfers, activity, time.time() - start_time)
+
+                for builder, transfer_paths in transfers.items():
+                    transfertool_obj = builder.make_transfertool(logger=logger, **transfertool_kwargs.get(builder.transfertool_class, {}))
                     start_time = time.time()
-
-                    transfertool_kwargs = {
-                        FTS3Transfertool: {
-                            'group_policy': group_policy,
-                            'group_bulk': group_bulk,
-                            'source_strategy': source_strategy,
-                            'max_time_in_queue': max_time_in_queue,
-                            'bring_online': bring_online,
-                            'default_lifetime': 172800,
-                            'archive_timeout_override': archive_timeout_override,
-                        },
-                        GlobusTransferTool: {
-                            'group_policy': transfertype,
-                            'group_bulk': group_bulk,
-                        },
-                    }
-                    transfers = transfer_core.next_transfers_to_submit(
-                        total_workers=heart_beat['nr_threads'],
-                        worker_number=heart_beat['assign_thread'],
-                        partition_hash_var=partition_hash_var,
-                        failover_schemes=failover_scheme,
-                        limit=bulk,
-                        activity=activity,
-                        rses=rse_ids,
-                        schemes=scheme,
-                        filter_transfertool=filter_transfertool,
-                        transfertools_by_name={transfertool: TRANSFERTOOL_CLASSES_BY_NAME[transfertool]},
-                        older_than=None,
-                        request_type=RequestType.TRANSFER,
-                        ignore_availability=ignore_availability,
-                        logger=logger,
-                    )
-                    total_transfers = len(list(hop for paths in transfers.values() for path in paths for hop in path))
-
-                    record_timer('daemons.conveyor.transfer_submitter.get_transfers.per_transfer', (time.time() - start_time) * 1000 / (total_transfers or 1))
-                    GET_TRANSFERS_COUNTER.inc(total_transfers)
-                    record_timer('daemons.conveyor.transfer_submitter.get_transfers.transfers', total_transfers)
-                    logger(logging.INFO, 'Got %s transfers for %s in %s seconds', total_transfers, activity, time.time() - start_time)
-
-                    for builder, transfer_paths in transfers.items():
-                        transfertool_obj = builder.make_transfertool(logger=logger, **transfertool_kwargs.get(builder.transfertool_class, {}))
-                        start_time = time.time()
-                        logger(logging.INFO, 'Starting to group transfers for %s (%s)', activity, transfertool_obj)
-                        grouped_jobs = transfertool_obj.group_into_submit_jobs(transfer_paths)
-                        record_timer('daemons.conveyor.transfer_submitter.bulk_group_transfer', (time.time() - start_time) * 1000 / (len(transfer_paths) or 1))
-
-                        logger(logging.INFO, 'Starting to submit transfers for %s (%s)', activity, transfertool_obj)
-                        for job in grouped_jobs:
-                            logger(logging.DEBUG, 'submitjob: transfers=%s, job_params=%s' % ([str(t) for t in job['transfers']], job['job_params']))
-                            submit_transfer(transfertool_obj=transfertool_obj, transfers=job['transfers'], job_params=job['job_params'], submitter='transfer_submitter',
-                                            timeout=timeout, logger=logger)
-
-                    if total_transfers < group_bulk:
-                        logger(logging.INFO, 'Only %s transfers for %s which is less than group bulk %s, sleep %s seconds', total_transfers, activity, group_bulk, sleep_time)
-                        if activity_next_exe_time[activity] < time.time():
-                            activity_next_exe_time[activity] = time.time() + sleep_time
-                except Exception:
-                    logger(logging.CRITICAL, 'Exception', exc_info=True)
-                    if once:
-                        raise
-
-            if once:
-                break
+                    logger(logging.DEBUG, 'Starting to group transfers for %s (%s)', activity, transfertool_obj)
+                    grouped_jobs = transfertool_obj.group_into_submit_jobs(transfer_paths)
+                    record_timer('daemons.conveyor.transfer_submitter.bulk_group_transfer', (time.time() - start_time) * 1000 / (len(transfer_paths) or 1))
+
+                    logger(logging.DEBUG, 'Starting to submit transfers for %s (%s)', activity, transfertool_obj)
+                    for job in grouped_jobs:
+                        logger(logging.DEBUG, 'submitjob: transfers=%s, job_params=%s' % ([str(t) for t in job['transfers']], job['job_params']))
+                        submit_transfer(transfertool_obj=transfertool_obj, transfers=job['transfers'], job_params=job['job_params'], submitter='transfer_submitter',
+                                        timeout=timeout, logger=logger)
+
+                if not once and total_transfers < group_bulk:
+                    logger(logging.DEBUG, 'Only %s transfers for %s which is less than group bulk %s, sleep %s seconds', total_transfers, activity, group_bulk, sleep_time)
+                    activity_next_exe_time[activity] = time.time() + sleep_time
+
+            except Exception:
+                logger(logging.CRITICAL, 'Exception', exc_info=True)
+                if once:
+                    raise
 
 
 def stop(signum=None, frame=None):


Test patch: