Repository: rucio/rucio

Issue:
client: dynamic bash completion
Motivation
----------
Bash completion (e.g., hitting `<tab>`) provides a powerful way to improve the the user experience.

With `argcomplete`, most of the static suggestions are present.  However, dynamic items like `rucio list-rse-attributes <tab>` (which should suggest available RSEs) don't work.

Let's add some smarter auto-complete logic!


Discussion:

Resolving patch:
diff --git a/lib/rucio/client/baseclient.py b/lib/rucio/client/baseclient.py
--- a/lib/rucio/client/baseclient.py
+++ b/lib/rucio/client/baseclient.py
@@ -12,6 +12,7 @@
 # - Ralph Vigne, <ralph.vigne@cern.ch>, 2015
 # - Martin Barisits, <martin.barisits@cern.ch>, 2017
 # - Mario Lassnig, <mario.lassnig@cern.ch>, 2017
+# - Brian Bockelman, <bbockelm@cse.unl.edu>, 2017
 #
 # Client class for callers of the Rucio system
 
@@ -25,7 +26,7 @@
 from rucio import version
 
 from logging import getLogger, StreamHandler, ERROR
-from os import environ, fdopen, path, makedirs
+from os import environ, fdopen, path, makedirs, geteuid
 from shutil import move
 from tempfile import mkstemp
 from urlparse import urlparse
@@ -136,7 +137,22 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None,
                     self.creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_cert'))))
                     self.creds['client_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_key'))))
                 elif self.auth_type == 'x509_proxy':
-                    self.creds['client_proxy'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_x509_proxy'))))
+                    try:
+                        self.creds['client_proxy'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_x509_proxy'))))
+                    except NoOptionError as error:
+                        # Recreate the classic GSI logic for locating the proxy:
+                        # - $X509_USER_PROXY, if it is set.
+                        # - /tmp/x509up_u`id -u` otherwise.
+                        # If neither exists (at this point, we don't care if it exists but is invalid), then rethrow
+                        if 'X509_USER_PROXY' in environ:
+                            self.creds['client_proxy'] = environ['X509_USER_PROXY']
+                        else:
+                            fname = '/tmp/x509up_u%d' % geteuid()
+                            if path.exists(fname):
+                                self.creds['client_proxy'] = path
+                            else:
+                                raise MissingClientParameter('Cannot find a valid X509 proxy; not in %s, $X509_USER_PROXY not set, and '
+                                                             '\'x509_proxy\' not set in the configuration file.' % fname)
                 elif self.auth_type == 'ssh':
                     self.creds['ssh_private_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'ssh_private_key'))))
             except (NoOptionError, NoSectionError) as error:
diff --git a/lib/rucio/extensions/forecast.py b/lib/rucio/extensions/forecast.py
--- a/lib/rucio/extensions/forecast.py
+++ b/lib/rucio/extensions/forecast.py
@@ -12,6 +12,7 @@
 Forecast library to predict the Transfers Time To Complete (T3C)
 """
 
+import datetime as dt
 import json
 import logging
 import time
@@ -21,8 +22,6 @@
 from rucio.db.sqla.session import read_session
 from rucio.db.sqla import models
 
-from sqlalchemy.sql.expression import and_
-
 
 def ewma(x_values, span):
     """
@@ -160,6 +159,8 @@ def predict_n(self, src, dst, size):
         :param size: size in bytes of the transfer.
         :returns: Number of seconds the transfer is going to take.
         """
+        src = self.rse2site(src)
+        dst = self.rse2site(dst)
         link = '__'.join([src, dst])
         link_true, rate, overhead, diskrw = self.recover_params(src, dst)
         if link_true == 'STANDARD_LINK':
@@ -171,7 +172,7 @@ def predict_n(self, src, dst, size):
 
     # Queue time prediction
     @read_session
-    def get_submitted_at_to_rucio(self, src, dst, act, session=None):
+    def get_submitted_at_rucio(self, src, dst, act, session=None):
         """
         Get the latest active transfer for the link from Rucio DB
 
@@ -184,21 +185,15 @@ def get_submitted_at_to_rucio(self, src, dst, act, session=None):
         site_dst = self.rse2site(dst)
         rses_id_src = self.site2rseids(site_src)
         rses_id_dst = self.site2rseids(site_dst)
-        submitted_times = session.query(models.Request.submitted_at).filter(
-            and_(and_(models.Request.source_rse_id in rses_id_src,
-                      models.Request.dest_rse_id in rses_id_dst),
-                 models.Request.activity == act))
+        submitted_times = session.query(models.Request.submitted_at)\
+                                 .filter(models.Request.state == models.RequestState.SUBMITTED,
+                                         models.Request.source_rse_id.in_(rses_id_src),
+                                         models.Request.dest_rse_id.in_(rses_id_dst),
+                                         models.Request.activity == act).all()
+        submitted_times = [(datetuple[0] - dt.datetime.utcfromtimestamp(0)).total_seconds() for datetuple in submitted_times]
+        submitted_times.sort(reverse=True)
         return submitted_times
 
-    def get_submitted_at_random(self, src, dst, act):
-        """
-        Generate a random sample of active transfers for the link.
-        This function will be removed when get_submitted_at_to_rucio is in place
-        """
-        submitted = [np.random.randint(time.time() - 7 * 24 * 60 * 60,
-                                       time.time()) for i in xrange(150)]
-        return submitted
-
     def predict_q(self, src, dst, act):
         """
         Make a prediction for the queue time
@@ -209,8 +204,18 @@ def predict_q(self, src, dst, act):
         :param act: The activity of the transfer.
         :returns: Number of seconds the transfer is going to spend in FTS queue.
         """
-        submitted = self.get_submitted_at_random(src, dst, act)
-        submitted.sort(reverse=True)
+        start = time.time()
+        submitted = self.get_submitted_at_rucio(src, dst, act)
+        if len(submitted) == 0:
+            submitted = [time.time()]
+        debug = True
+        if debug is True:
+            logging.warning('Query took ' + str(time.time() - start) + ' seconds to retrieve (' + str(len(submitted)) + ') rows.')
+            f = open('/tmp/forecast.log', 'a')
+            f.write(src + ' --> ' + dst + ' (' + act + ') time:' + str(time.time() - start) + '\n')
+            f.write(str(submitted))
+            f.write('\n')
+            f.close()
         submitted = np.array(submitted)
         submitted = int(time.time()) - submitted
         return ewma(submitted, span=10)[-1]
@@ -220,14 +225,13 @@ def predict(self, transfers):
         """
         Make a prediction for queue and network time for a collection of transfers
 
-        :param transfers: A list of dictionaries, each of wich at least have the keys src: The name of the source RSE for the transfer.
-                          dst: The name of the destination RSE for the transfer. act: The activity of the transfer. size: The size in bytes of the transfer.
+        :param transfers: A list of dictionaries, each of wich at least have the keys src: The name of the source RSE for the transfer. dst: The name of the destination RSE for the transfer. act: The activity of the transfer. size: The size in bytes.
         :returns: A list of the previous dictionaries extended with two new keys ntime: Number of seconds the transfer is going to spend in in the network. qtime: Number of seconds the transfer is going to spend in FTS queue.
         """
         result = []
         for transfer in transfers:
-            src = self.rse2site(transfer['src'])
-            dst = self.rse2site(transfer['dst'])
+            src = transfer['src']
+            dst = transfer['dst']
             act = transfer['activity']
             size = transfer['size']
             transfer['ntime'] = self.predict_n(src, dst, size)
diff --git a/lib/rucio/web/rest/redirect.py b/lib/rucio/web/rest/redirect.py
--- a/lib/rucio/web/rest/redirect.py
+++ b/lib/rucio/web/rest/redirect.py
@@ -176,7 +176,7 @@ def GET(self, scope, name):
         try:
 
             # use the default HTTP protocols if no scheme is given
-            select, rse, site, schemes = 'random', None, None, ['http', 'https', 's3+rucio']
+            select, rse, site, schemes = 'random', None, None, ['davs', 'http', 'https', 's3+rucio']
 
             client_ip = ctx.env.get('HTTP_X_FORWARDED_FOR')
             if client_ip is None:


Test patch:
diff --git a/lib/rucio/tests/test_forecast.py b/lib/rucio/tests/test_forecast.py
--- a/lib/rucio/tests/test_forecast.py
+++ b/lib/rucio/tests/test_forecast.py
@@ -8,17 +8,31 @@
 # Authors:
 # - Joaquin Bogado, <jbogadog@cern.ch>, 2017
 
+from rucio.db.sqla import models
+from rucio.db.sqla.session import read_session
 from rucio.extensions.forecast import T3CModel
 
+from random import choice
+
+
+@read_session
+def get_session(session=None):
+    return session
+
 
 class TestForecast():
     """
     Class to test the T3C prediction clases
     """
-
     def test_predict(self):
+        session = get_session()
+        src = choice(session.query(models.RSE).all()).rse
+        dst = choice(session.query(models.RSE).all()).rse
+        act = 'Production Input'
+        size = 10**9
+
         model = T3CModel()
-        data = [{'src': 'BNL-OSG2_DATADISK', 'dst': 'MWT2_DATADISK',
-                 'activity': 'Production_Input', 'size': 213427540000}]
+
+        data = [{'src': src, 'dst': dst, 'activity': act, 'size': size}]
         result = model.predict(data)
         return result[0]['ntime'] > 0 and result[0]['qtime'] > 0