Repository: rucio/rucio

Issue:
Deprecate conveyor-poller-latest
Motivation
----------
The poller-latest is not used anymore and only the normale conveyor-poller should be used.

Modification
------------
Remove the poller-latest and exclusively used code of it.




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -812,41 +812,6 @@ def set_transfer_update_time(external_host, transfer_id, update_time=datetime.da
         raise UnsupportedOperation("Transfer %s doesn't exist or its status is not submitted." % transfer_id)
 
 
-def query_latest(external_host, state, last_nhours=1, logger=logging.log):
-    """
-    Query the latest transfers in last n hours with state.
-    :param external_host:  FTS host name as a string.
-    :param state:          FTS job state as a string or a dictionary.
-    :param last_nhours:    Latest n hours as an integer.
-    :param logger:         Optional decorated logger that can be passed from the calling daemons or servers.
-    :returns:              Requests status information as a dictionary.
-    """
-
-    record_counter('core.request.query_latest')
-
-    start_time = time.time()
-    resps = FTS3Transfertool(external_host=external_host).query_latest(state=state, last_nhours=last_nhours)
-    record_timer('core.request.query_latest_fts3.%s.%s_hours' % (external_host, last_nhours), (time.time() - start_time) * 1000)
-
-    if not resps:
-        return
-
-    ret_resps = []
-    for resp in resps:
-        if 'job_metadata' not in resp or resp['job_metadata'] is None or 'issuer' not in resp['job_metadata'] or resp['job_metadata']['issuer'] != 'rucio':
-            continue
-
-        if 'request_id' not in resp['job_metadata']:
-            # submitted by new submitter
-            try:
-                logger(logging.DEBUG, "Transfer %s on %s is %s, decrease its updated_at." % (resp['job_id'], external_host, resp['job_state']))
-                set_transfer_update_time(external_host, resp['job_id'], datetime.datetime.utcnow() - datetime.timedelta(hours=24))
-            except Exception as error:
-                logger(logging.DEBUG, "Exception happened when updating transfer updatetime: %s" % str(error).replace('\n', ''))
-
-    return ret_resps
-
-
 @transactional_session
 def touch_transfer(external_host, transfer_id, session=None):
     """
diff --git a/lib/rucio/daemons/conveyor/poller_latest.py b/lib/rucio/daemons/conveyor/poller_latest.py
deleted file mode 100644
--- a/lib/rucio/daemons/conveyor/poller_latest.py
+++ /dev/null
@@ -1,165 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015-2021 CERN
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Authors:
-# - Wen Guan <wen.guan@cern.ch>, 2015-2016
-# - Vincent Garonne <vincent.garonne@cern.ch>, 2015-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2016-2017
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2018
-# - Thomas Beermann <thomas.beermann@cern.ch>, 2020-2021
-# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2021
-# - Sahan Dilshan <32576163+sahandilshan@users.noreply.github.com>, 2021
-# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
-
-"""
-Conveyor is a daemon to manage file transfers.
-"""
-
-import datetime
-import logging
-import os
-import socket
-import threading
-import time
-import traceback
-
-from requests.exceptions import RequestException
-
-import rucio.db.sqla.util
-from rucio.common import exception
-from rucio.common.constants import FTS_STATE
-from rucio.common.logging import setup_logging
-from rucio.common.utils import daemon_sleep
-from rucio.core import heartbeat, transfer, request
-from rucio.core.monitor import record_timer, record_counter
-
-graceful_stop = threading.Event()
-
-datetime.datetime.strptime('', '')
-
-
-def poller_latest(external_hosts, once=False, last_nhours=1, fts_wait=1800, sleep_time=1800):
-    """
-    Main loop to check the status of a transfer primitive with a transfertool.
-    :param fts_wait: OBSOLETE, please use sleep_time instead
-    """
-    if sleep_time == poller_latest.__defaults__[3] and fts_wait != poller_latest.__defaults__[2]:
-        sleep_time = fts_wait
-    executable = 'conveyor-poller-latest'
-    hostname = socket.getfqdn()
-    pid = os.getpid()
-    hb_thread = threading.current_thread()
-
-    logging.info('polling latest %s hours on hosts: %s' % (last_nhours, external_hosts))
-    if external_hosts:
-        if isinstance(external_hosts, str):
-            external_hosts = [external_hosts]
-
-    while not graceful_stop.is_set():
-
-        try:
-            heartbeat.live(executable, hostname, pid, hb_thread)
-
-            start_time = time.time()
-            for external_host in external_hosts:
-                logging.debug('polling latest %s hours on host: %s' % (last_nhours, external_host))
-                ts = time.time()
-                resps = None
-                state = [FTS_STATE.FINISHED,
-                         FTS_STATE.FAILED,
-                         FTS_STATE.FINISHEDDIRTY,
-                         FTS_STATE.CANCELED]
-                try:
-                    resps = transfer.query_latest(external_host, state=state, last_nhours=last_nhours)
-                except Exception:
-                    logging.error(traceback.format_exc())
-                record_timer('daemons.conveyor.poller_latest.000-query_latest', (time.time() - ts) * 1000)
-
-                if resps:
-                    logging.info('poller_latest - polling %i requests' % (len(resps)))
-
-                if not resps or resps == []:
-                    if once:
-                        break
-                    logging.info("no requests found. will sleep 60 seconds")
-                    continue
-
-                for resp in resps:
-                    try:
-                        ret = request.update_request_state(resp)
-                        # if True, really update request content; if False, only touch request
-                        record_counter('daemons.conveyor.poller_latest.update_request_state.%s' % ret)
-                    except Exception:
-                        logging.error(traceback.format_exc())
-            if once:
-                break
-
-            daemon_sleep(start_time=start_time, sleep_time=sleep_time, graceful_stop=graceful_stop)
-        except RequestException as error:
-            logging.error("Failed to contact FTS server: %s" % (str(error)))
-        except Exception:
-            logging.critical(traceback.format_exc())
-
-        if once:
-            return
-
-    logging.info('poller_latest - graceful stop requests')
-
-    heartbeat.die(executable, hostname, pid, hb_thread)
-
-    logging.info('poller_latest - graceful stop done')
-
-
-def stop(signum=None, frame=None):
-    """
-    Graceful exit.
-    """
-
-    graceful_stop.set()
-
-
-def run(once=False, last_nhours=1, external_hosts=None, fts_wait=1800, total_threads=1, sleep_time=1800):
-    """
-    Starts up the conveyer threads.
-    :param fts_wait: OBSOLETE, please use sleep_time instead
-    """
-    setup_logging()
-
-    if rucio.db.sqla.util.is_old_db():
-        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
-
-    if not external_hosts:
-        external_hosts = []
-
-    if once:
-        logging.info('executing one poller iteration only')
-        poller_latest(external_hosts, once=once, last_nhours=last_nhours)
-    else:
-
-        logging.info('starting poller threads')
-
-        threads = [threading.Thread(target=poller_latest, kwargs={'external_hosts': external_hosts,
-                                                                  'fts_wait': fts_wait,
-                                                                  'last_nhours': last_nhours,
-                                                                  'sleep_time': sleep_time}) for _ in range(0, total_threads)]
-
-        [thread.start() for thread in threads]
-
-        logging.info('waiting for interrupts')
-
-        # Interruptible joins require a timeout.
-        while threads:
-            threads = [thread.join(timeout=3.14) for thread in threads if thread and thread.is_alive()]
diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py
--- a/lib/rucio/transfertool/fts3.py
+++ b/lib/rucio/transfertool/fts3.py
@@ -74,7 +74,6 @@
 QUERY_COUNTER = Counter('rucio_transfertool_fts3_query', 'Number of queried transfers', labelnames=('state', 'host'))
 WHOAMI_COUNTER = Counter('rucio_transfertool_fts3_whoami', 'Number of whoami requests', labelnames=('state', 'host'))
 VERSION_COUNTER = Counter('rucio_transfertool_fts3_version', 'Number of version requests', labelnames=('state', 'host'))
-QUERY_LATEST_COUNTER = Counter('rucio_transfertool_fts3_query_latest', 'Number of latest status queries', labelnames=('state', 'host'))
 BULK_QUERY_COUNTER = Counter('rucio_transfertool_fts3_bulk_query', 'Number of bulk queries', labelnames=('state', 'host'))
 QUERY_DETAILS_COUNTER = Counter('rucio_transfertool_fts3_query_details', 'Number of detailed status queries', labelnames=('state', 'host'))
 SUBMISSION_TIMER = Summary('rucio_transfertool_fts3_submit_transfer', 'Timer for transfer submission', labelnames=('host',))
@@ -349,58 +348,6 @@ def version(self):
         VERSION_COUNTER.labels(**labels).inc()
         raise Exception('Could not retrieve version: %s', get_result.content)
 
-    def query_latest(self, state, last_nhours=1):
-        """
-        Query the latest status transfers status in FTS3 via JSON.
-
-        :param state: Transfer state as a string or a dictionary.
-        :returns: Transfer status information as a dictionary.
-        """
-
-        jobs = None
-
-        try:
-            whoami = requests.get('%s/whoami' % (self.external_host),
-                                  verify=self.verify,
-                                  cert=self.cert,
-                                  headers=self.headers)
-            if whoami and whoami.status_code == 200:
-                delegation_id = whoami.json()['delegation_id']
-            else:
-                raise Exception('Could not retrieve delegation id: %s', whoami.content)
-            state_string = ','.join(state)
-            jobs = requests.get('%s/jobs?dlg_id=%s&state_in=%s&time_window=%s' % (self.external_host,
-                                                                                  delegation_id,
-                                                                                  state_string,
-                                                                                  last_nhours),
-                                verify=self.verify,
-                                cert=self.cert,
-                                headers=self.headers)
-        except ReadTimeout as error:
-            raise TransferToolTimeout(error)
-        except JSONDecodeError as error:
-            raise TransferToolWrongAnswer(error)
-        except Exception:
-            logging.warning('Could not query latest terminal states from %s', self.external_host)
-
-        if jobs and (jobs.status_code == 200 or jobs.status_code == 207):
-            record_counter('transfertool.fts3.%s.query_latest.success' % self.__extract_host(self.external_host))
-            labels = {'state': 'success', 'host': self.__extract_host(self.external_host)}
-            QUERY_LATEST_COUNTER.labels(**labels).inc()
-            try:
-                jobs_json = jobs.json()
-                return jobs_json
-            except ReadTimeout as error:
-                raise TransferToolTimeout(error)
-            except JSONDecodeError as error:
-                raise TransferToolWrongAnswer(error)
-            except Exception as error:
-                logging.error("Failed to parse the jobs status %s" % (str(error)))
-
-        record_counter('transfertool.fts3.%s.query.failure' % self.__extract_host(self.external_host))
-        labels = {'state': 'failure', 'host': self.__extract_host(self.external_host)}
-        QUERY_LATEST_COUNTER.labels(**labels).inc()
-
     def bulk_query(self, transfer_ids, timeout=None):
         """
         Query the status of a bulk of transfers in FTS3 via JSON.


Test patch:
diff --git a/lib/rucio/tests/test_daemons.py b/lib/rucio/tests/test_daemons.py
--- a/lib/rucio/tests/test_daemons.py
+++ b/lib/rucio/tests/test_daemons.py
@@ -29,7 +29,7 @@
 from rucio.daemons.badreplicas import minos, minos_temporary_expiration, necromancer
 from rucio.daemons.c3po import c3po
 from rucio.daemons.cache import consumer
-from rucio.daemons.conveyor import finisher, fts_throttler, poller, poller_latest, receiver, stager, submitter, throttler, preparer
+from rucio.daemons.conveyor import finisher, fts_throttler, poller, receiver, stager, submitter, throttler, preparer
 from rucio.daemons.follower import follower
 from rucio.daemons.hermes import hermes, hermes2
 from rucio.daemons.judge import cleaner, evaluator, injector, repairer
@@ -61,7 +61,6 @@
     finisher,
     fts_throttler,
     poller,
-    poller_latest,
     receiver,
     stager,
     submitter,