Repository: rucio/rucio

Issue:
Caching collisions with rse_attributes and rse_info
Motivation
----------
There is a caching collision with the rse_info values as well as the rse_attributes in the conveyor. Both dictionaries are stored in memcache under the `rse_id` key, thus leading to collisions.

https://github.com/rucio/rucio/blob/e7c4def359e1aeb413966e383b83ac9887f3c728/lib/rucio/rse/rsemanager.py#L96

https://github.com/rucio/rucio/blob/e7c4def359e1aeb413966e383b83ac9887f3c728/lib/rucio/common/rse_attributes.py#L52

Modification
------------
Use proper keys for both.



Discussion:
This should also be back-ported to LTS releases.

Resolving patch:
diff --git a/lib/rucio/common/rse_attributes.py b/lib/rucio/common/rse_attributes.py
--- a/lib/rucio/common/rse_attributes.py
+++ b/lib/rucio/common/rse_attributes.py
@@ -1,13 +1,25 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# -*- coding: utf-8 -*-
+# Copyright 2015-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
+# you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# http://www.apache.org/licenses/LICENSE-2.0OA
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # Authors:
-# - Wen Guan, <wen.guan@cern.ch>, 2015
-# - Brandon White, <bjwhite@fnal.gov>, 2019
+# - Wen Guan <wen.guan@cern.ch>, 2015
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2017
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
+# - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
+# - Brandon White <bjwhite@fnal.gov>, 2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -28,10 +40,6 @@
                                  expiration_time=3600,
                                  arguments={'url': config_get('cache', 'url', False, '127.0.0.1:11211'), 'distributed_lock': True})
 
-# for local test
-# REGION = make_region().configure('dogpile.cache.memory',
-#                                  expiration_time=3600)
-
 
 def get_rse_attributes(rse_id, session=None):
     """
@@ -44,12 +52,13 @@ def get_rse_attributes(rse_id, session=None):
     :returns: A dictionary with RSE attributes for a RSE.
     """
 
-    result = REGION.get(rse_id)
+    key = 'rse_attributes_%s' % (rse_id)
+    result = REGION.get(key)
     if isinstance(result, NoValue):
         try:
             result = None
             result = rse_core.list_rse_attributes(rse_id=rse_id, session=session)
-            REGION.set(rse_id, result)
+            REGION.set(key, result)
         except:
             logging.warning("Failed to get RSE %s attributes, error: %s" % (rse_id, traceback.format_exc()))
     return result
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -1,4 +1,5 @@
-# Copyright 2012-2020 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2012-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +16,7 @@
 # Authors:
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2012-2015
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2018
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2020
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2012-2019
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
 # - Wen Guan <wen.guan@cern.ch>, 2014-2017
@@ -23,7 +24,7 @@
 # - Tobias Wegner <twegner@cern.ch>, 2017-2019
 # - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
 # - Frank Berghaus <frank.berghaus@cern.ch>, 2018-2019
-# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
+# - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Nicolo Magini <nicolo.magini@cern.ch>, 2018
 # - Tomas Javurek <tomas.javurek@cern.ch>, 2018-2020
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
@@ -90,6 +91,7 @@ def get_rse_info(rse=None, vo='def', rse_id=None, session=None):
     # __request_rse_info will be assigned when the module is loaded as it depends on the rucio environment (server or client)
     # __request_rse_info, rse_region are defined in /rucio/rse/__init__.py
     key = '{}:{}'.format(rse, vo) if rse_id is None else str(rse_id)
+    key = 'rse_info_%s' % (key)
     rse_info = RSE_REGION.get(key)   # NOQA pylint: disable=undefined-variable
     if not rse_info:  # no cached entry found
         rse_info = __request_rse_info(str(rse), vo=vo, rse_id=rse_id, session=session)  # NOQA pylint: disable=undefined-variable


Test patch: