Repository: rucio/rucio

Issue:
get_submitted_at_rucio should return an array of submitted_at time stamps
Motivation
----------
get_submitted_at_rucio function in forecast.py extension is buggy. 

Modification
------------
The function have access to rucio database, but the query is tricky and need testing on production to see if the time to retrieve the submitted_at time stamps for a link in not an overkill.

Expected behavior
-----------------
Should return an array on time stamps ordered descendent (the oldest one at the end). 


Discussion:

Resolving patch:
diff --git a/lib/rucio/extensions/forecast.py b/lib/rucio/extensions/forecast.py
--- a/lib/rucio/extensions/forecast.py
+++ b/lib/rucio/extensions/forecast.py
@@ -12,6 +12,7 @@
 Forecast library to predict the Transfers Time To Complete (T3C)
 """
 
+import datetime as dt
 import json
 import logging
 import time
@@ -21,8 +22,6 @@
 from rucio.db.sqla.session import read_session
 from rucio.db.sqla import models
 
-from sqlalchemy.sql.expression import and_
-
 
 def ewma(x_values, span):
     """
@@ -160,6 +159,8 @@ def predict_n(self, src, dst, size):
         :param size: size in bytes of the transfer.
         :returns: Number of seconds the transfer is going to take.
         """
+        src = self.rse2site(src)
+        dst = self.rse2site(dst)
         link = '__'.join([src, dst])
         link_true, rate, overhead, diskrw = self.recover_params(src, dst)
         if link_true == 'STANDARD_LINK':
@@ -171,7 +172,7 @@ def predict_n(self, src, dst, size):
 
     # Queue time prediction
     @read_session
-    def get_submitted_at_to_rucio(self, src, dst, act, session=None):
+    def get_submitted_at_rucio(self, src, dst, act, session=None):
         """
         Get the latest active transfer for the link from Rucio DB
 
@@ -184,21 +185,15 @@ def get_submitted_at_to_rucio(self, src, dst, act, session=None):
         site_dst = self.rse2site(dst)
         rses_id_src = self.site2rseids(site_src)
         rses_id_dst = self.site2rseids(site_dst)
-        submitted_times = session.query(models.Request.submitted_at).filter(
-            and_(and_(models.Request.source_rse_id in rses_id_src,
-                      models.Request.dest_rse_id in rses_id_dst),
-                 models.Request.activity == act))
+        submitted_times = session.query(models.Request.submitted_at)\
+                                 .filter(models.Request.state == models.RequestState.SUBMITTED,
+                                         models.Request.source_rse_id.in_(rses_id_src),
+                                         models.Request.dest_rse_id.in_(rses_id_dst),
+                                         models.Request.activity == act).all()
+        submitted_times = [(datetuple[0] - dt.datetime.utcfromtimestamp(0)).total_seconds() for datetuple in submitted_times]
+        submitted_times.sort(reverse=True)
         return submitted_times
 
-    def get_submitted_at_random(self, src, dst, act):
-        """
-        Generate a random sample of active transfers for the link.
-        This function will be removed when get_submitted_at_to_rucio is in place
-        """
-        submitted = [np.random.randint(time.time() - 7 * 24 * 60 * 60,
-                                       time.time()) for i in xrange(150)]
-        return submitted
-
     def predict_q(self, src, dst, act):
         """
         Make a prediction for the queue time
@@ -209,8 +204,18 @@ def predict_q(self, src, dst, act):
         :param act: The activity of the transfer.
         :returns: Number of seconds the transfer is going to spend in FTS queue.
         """
-        submitted = self.get_submitted_at_random(src, dst, act)
-        submitted.sort(reverse=True)
+        start = time.time()
+        submitted = self.get_submitted_at_rucio(src, dst, act)
+        if len(submitted) == 0:
+            submitted = [time.time()]
+        debug = True
+        if debug is True:
+            logging.warning('Query took ' + str(time.time() - start) + ' seconds to retrieve (' + str(len(submitted)) + ') rows.')
+            f = open('/tmp/forecast.log', 'a')
+            f.write(src + ' --> ' + dst + ' (' + act + ') time:' + str(time.time() - start) + '\n')
+            f.write(str(submitted))
+            f.write('\n')
+            f.close()
         submitted = np.array(submitted)
         submitted = int(time.time()) - submitted
         return ewma(submitted, span=10)[-1]
@@ -220,14 +225,13 @@ def predict(self, transfers):
         """
         Make a prediction for queue and network time for a collection of transfers
 
-        :param transfers: A list of dictionaries, each of wich at least have the keys src: The name of the source RSE for the transfer.
-                          dst: The name of the destination RSE for the transfer. act: The activity of the transfer. size: The size in bytes of the transfer.
+        :param transfers: A list of dictionaries, each of wich at least have the keys src: The name of the source RSE for the transfer. dst: The name of the destination RSE for the transfer. act: The activity of the transfer. size: The size in bytes.
         :returns: A list of the previous dictionaries extended with two new keys ntime: Number of seconds the transfer is going to spend in in the network. qtime: Number of seconds the transfer is going to spend in FTS queue.
         """
         result = []
         for transfer in transfers:
-            src = self.rse2site(transfer['src'])
-            dst = self.rse2site(transfer['dst'])
+            src = transfer['src']
+            dst = transfer['dst']
             act = transfer['activity']
             size = transfer['size']
             transfer['ntime'] = self.predict_n(src, dst, size)
diff --git a/lib/rucio/vcsversion.py b/lib/rucio/vcsversion.py
--- a/lib/rucio/vcsversion.py
+++ b/lib/rucio/vcsversion.py
@@ -4,8 +4,8 @@
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '1.14.1',
-    'branch_nick': 'patch-0-Preparation_for_1_14_1_release',
-    'revision_id': '94fc77cad3154b5807647c138674d959e6a029bb',
-    'revno': 5674
+    'version': '1.14.2',
+    'branch_nick': 'patch-0-1_14_2_preparation',
+    'revision_id': '4e6e3c347088ff2d2959c077bf9dcf91a06a5bed',
+    'revno': 5697
 }
diff --git a/setup_rucio.py b/setup_rucio.py
--- a/setup_rucio.py
+++ b/setup_rucio.py
@@ -113,6 +113,14 @@ def write_requirements():
         req_file.close()
 
 
+oracle_extras = ['cx_oracle>=5.1']
+postgresql_extras = ['psycopg2>=2.4.2']
+mysql_extras = ['PyMySQL']
+
+requires = parse_requirements(requirements_files=requirements_files)
+extras_require = dict(oracle=oracle_extras,
+                      postgresql=postgresql_extras,
+                      mysql=mysql_extras)
 requires = parse_requirements(requirements_files=requirements_files)
 depend_links = parse_dependency_links(requirements_files=requirements_files)
 
@@ -163,5 +171,6 @@ def get_file_list(self):
         'Programming Language :: Python :: 2.7',
         'Environment :: No Input/Output (Daemon)', ],
     install_requires=requires,
+    extras_require=extras_require,
     dependency_links=depend_links,
 )


Test patch:
diff --git a/lib/rucio/tests/test_forecast.py b/lib/rucio/tests/test_forecast.py
--- a/lib/rucio/tests/test_forecast.py
+++ b/lib/rucio/tests/test_forecast.py
@@ -8,17 +8,31 @@
 # Authors:
 # - Joaquin Bogado, <jbogadog@cern.ch>, 2017
 
+from rucio.db.sqla import models
+from rucio.db.sqla.session import read_session
 from rucio.extensions.forecast import T3CModel
 
+from random import choice
+
+
+@read_session
+def get_session(session=None):
+    return session
+
 
 class TestForecast():
     """
     Class to test the T3C prediction clases
     """
-
     def test_predict(self):
+        session = get_session()
+        src = choice(session.query(models.RSE).all()).rse
+        dst = choice(session.query(models.RSE).all()).rse
+        act = 'Production Input'
+        size = 10**9
+
         model = T3CModel()
-        data = [{'src': 'BNL-OSG2_DATADISK', 'dst': 'MWT2_DATADISK',
-                 'activity': 'Production_Input', 'size': 213427540000}]
+
+        data = [{'src': src, 'dst': dst, 'activity': act, 'size': size}]
         result = model.predict(data)
         return result[0]['ntime'] > 0 and result[0]['qtime'] > 0