Repository: rucio/rucio

Issue:
fix caching problem in config core for running tests
Motivation
----------
Currently the tests in [https://github.com/rucio/rucio/blob/master/lib/rucio/tests/test_config.py#L19](https://github.com/rucio/rucio/blob/master/lib/rucio/tests/test_config.py#L19) are failing if you use a cache in your tests because you can read a stale value that got set during running the test due to a quite long expiration time.


Modification
------------
- force a readout from DB to get the current value
- add memcache to the travis test (because the PR that added caching went through because travis was not using any cache)


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/config.py b/lib/rucio/core/config.py
--- a/lib/rucio/core/config.py
+++ b/lib/rucio/core/config.py
@@ -14,27 +14,40 @@
   PY3K COMPATIBLE
 '''
 
+from dogpile.cache import make_region
+from dogpile.cache.api import NoValue
+
 from rucio.common.exception import ConfigNotFound
 from rucio.db.sqla import models
 from rucio.db.sqla.session import read_session, transactional_session
 
 
+REGION = make_region().configure('dogpile.cache.memcached',
+                                 expiration_time=3600,
+                                 arguments={'url': "127.0.0.1:11211", 'distributed_lock': True})
+
+
 @read_session
-def sections(session=None):
+def sections(use_cache=True, expiration_time=3600, session=None):
     """
     Return a list of the sections available.
 
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
     :param session: The database session in use.
     :returns: ['section_name', ...]
     """
 
-    all_sections = session.query(models.Config.section).distinct().all()
+    sections_key = 'sections'
+    all_sections = NoValue()
+    if use_cache:
+        all_sections = REGION.get(sections_key, expiration_time=expiration_time)
+    if isinstance(all_sections, NoValue):
+        query = session.query(models.Config.section).distinct().all()
+        all_sections = [section[0] for section in query]
+        REGION.set(sections_key, all_sections)
 
-    res = []
-    for section in all_sections:
-        res.append(section[0])
-
-    return res
+    return all_sections
 
 
 @transactional_session
@@ -49,57 +62,74 @@ def add_section(section, session=None):
 
 
 @read_session
-def has_section(section, session=None):
+def has_section(section, use_cache=True, expiration_time=3600, session=None):
     """
     Indicates whether the named section is present in the configuration.
 
     :param section: The name of the section.
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
     :param session: The database session in use.
     :returns: True/False
     """
-
-    query = session.query(models.Config).filter_by(section=section)
-
-    return True if query.first() else False
+    has_section_key = 'has_section_%s' % section
+    has_section = NoValue()
+    if use_cache:
+        has_section = REGION.get(has_section_key, expiration_time=expiration_time)
+    if isinstance(has_section, NoValue):
+        query = session.query(models.Config).filter_by(section=section)
+        has_section = True if query.first() else False
+        REGION.set(has_section_key, has_section)
+    return has_section
 
 
 @read_session
-def options(section, session=None):
+def options(section, use_cache=True, expiration_time=3600, session=None):
     """
     Returns a list of options available in the specified section.
 
     :param section: The name of the section.
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
     :param session: The database session in use.
     :returns: ['option', ...]
     """
-
-    optns = session.query(models.Config.opt).filter_by(section=section).distinct().all()
-
-    res = []
-    for optn in optns:
-        res.append(optn[0])
-
-    return res
+    options_key = 'options'
+    options = NoValue()
+    if use_cache:
+        options = REGION.get(options_key, expiration_time=expiration_time)
+    if isinstance(options, NoValue):
+        query = session.query(models.Config.opt).filter_by(section=section).distinct().all()
+        options = [option[0] for option in query]
+        REGION.set(options_key, options)
+    return options
 
 
 @read_session
-def has_option(section, option, session=None):
+def has_option(section, option, use_cache=True, expiration_time=3600, session=None):
     """
     Check if the given section exists and contains the given option.
 
     :param section: The name of the section.
     :param option: The name of the option.
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
     :param session: The database session in use.
     :returns: True/False
     """
-
-    query = session.query(models.Config).filter_by(section=section, opt=option)
-
-    return True if query.first() else False
+    has_option_key = 'has_option_%s_%s' % (section, option)
+    has_option = NoValue()
+    if use_cache:
+        has_option = REGION.get(has_option_key, expiration_time=expiration_time)
+    if isinstance(has_option, NoValue):
+        query = session.query(models.Config).filter_by(section=section, opt=option)
+        has_option = True if query.first() else False
+        REGION.set(has_option_key, has_option)
+    return has_option
 
 
 @read_session
-def get(section, option, default=None, session=None):
+def get(section, option, default=None, use_cache=True, expiration_time=3600, session=None):
     """
     Get an option value for the named section. Value can be auto-coerced to string, int, float, bool, None.
 
@@ -109,38 +139,48 @@ def get(section, option, default=None, session=None):
     :param section: The name of the section.
     :param option: The name of the option.
     :param default: The default value if no value is found.
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
     :param session: The database session in use.
     :returns: The auto-coerced value.
     """
-
-    tmp = session.query(models.Config.value).filter_by(section=section, opt=option).first()
-
-    if tmp is not None:
-        return __convert_type(tmp[0])
-    elif default is None:
-        raise ConfigNotFound
+    value_key = 'get_%s_%s' % (section, option)
+    value = NoValue()
+    if use_cache:
+        value = REGION.get(value_key, expiration_time=expiration_time)
+    if isinstance(value, NoValue):
+        tmp = session.query(models.Config.value).filter_by(section=section, opt=option).first()
+        if tmp is not None:
+            value = __convert_type(tmp[0])
+            REGION.set(value_key, tmp[0])
+        elif default is None:
+            raise ConfigNotFound
+        else:
+            value = default
     else:
-        return default
+        value = __convert_type(value)
+    return value
 
 
 @read_session
-def items(section, session=None):
+def items(section, use_cache=True, expiration_time=3600, session=None):
     """
     Return a list of (option, value) pairs for each option in the given section. Values are auto-coerced as in get().
 
     :param section: The name of the section.
-    :param value: The content of the value.
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
     :param session: The database session in use.
     :returns: [('option', auto-coerced value), ...]
     """
-
-    itms = session.query(models.Config.opt, models.Config.value).filter_by(section=section).all()
-
-    res = []
-    for itm in itms:
-        res.append((itm[0], __convert_type(itm[1])))
-
-    return res
+    items_key = 'items_%s' % section
+    items = NoValue()
+    if use_cache:
+        items = REGION.get(items_key, expiration_time=expiration_time)
+    if isinstance(items, NoValue):
+        items = session.query(models.Config.opt, models.Config.value).filter_by(section=section).all()
+        REGION.set(items_key, items)
+    return [(item[0], __convert_type(item[1])) for item in items]
 
 
 @transactional_session


Test patch:
diff --git a/lib/rucio/tests/test_config.py b/lib/rucio/tests/test_config.py
--- a/lib/rucio/tests/test_config.py
+++ b/lib/rucio/tests/test_config.py
@@ -23,7 +23,7 @@ def test_get_config_sections(self):
         expected_sections = [str(generate_uuid()), str(generate_uuid())]
         for section in expected_sections:
             config_core.set(section, str(generate_uuid()), str(generate_uuid()))
-        sections = config_core.sections()
+        sections = config_core.sections(use_cache=False)
         for section in expected_sections:
             assert_in(section, sections)
 
@@ -34,14 +34,14 @@ def test_get_and_set_section_option(self):
         option = str(generate_uuid())
         expected_value = str(generate_uuid())
         config_core.set(section=section, option=option, value=expected_value)
-        value = config_core.get(section, option)
+        value = config_core.get(section, option, use_cache=False)
         assert_equal(value, expected_value)
 
         # default value
         section = str(generate_uuid())
         config_core.set(section=section, option=str(generate_uuid()), value=str(generate_uuid()))
         default_value = 'default'
-        value = config_core.get(section, 'new_option', default=default_value)
+        value = config_core.get(section, 'new_option', default=default_value, use_cache=False)
         assert_equal(value, default_value)
 
 
diff --git a/tools/run_tests_docker.sh b/tools/run_tests_docker.sh
--- a/tools/run_tests_docker.sh
+++ b/tools/run_tests_docker.sh
@@ -16,7 +16,9 @@
 # Authors:
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2017-2018
 # - Vincent Garonne <vgaronne@gmail.com>, 2018
-# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
+
+memcached -u root -d
 
 noseopts="--exclude=test_dq2* --exclude=.*test_rse_protocol_.* --exclude=test_alembic --exclude=test_rucio_cache --exclude=test_rucio_server --exclude=test_objectstore --exclude=test_auditor*"