Repository: dmwm/wmcore
Issue: Datatier validation does not use the DbsUrl from the workload config

**Impact of the bug**
ReqMgr2

**Describe the bug**
The request dictionary contains a `DbsUrl` key, which is supposed to be used for the input data (plus parent and pileup) check, and also against any other validation needed against DBS (like the datatiers).
With the current problems in DBS testbed, I noticed that even if we point DbsUrl to the production instance, it still fails the datatier validation because it cannot find existent datatiers in the dbs api response.

The problem seems to be here:
https://github.com/dmwm/WMCore/blob/master/src/python/WMCore/ReqMgr/Utils/Validation.py#L263

which uses a single cache name, no matter the DbsUrl provided. Thus, once a cache gets created, it will always use the same until it gets expired...

**How to reproduce it**
Sort of using this gist
https://gist.github.com/amaltaro/2c546c9a251d12b2f7bfa3d04eb56e34

**Expected behavior**
We should have a different cache structure for each DBS instance. Such that we can fetch data from the correct cache.
That means we need to name the caches after the dbs url. We can likely hash the dbsurl and append it to the current name.

**Additional context and error message**
Follow up of: https://github.com/dmwm/WMCore/pull/9355

Changed files:
test/python/WMCore_t/Services_t/DBS_t/DBSReader_t.py, src/python/WMCore/ReqMgr/Utils/Validation.py, src/python/WMCore/Services/DBS/DBS3Reader.py, src/python/WMCore/Cache/GenericDataCache.py

diff --git a/src/python/WMCore/Cache/GenericDataCache.py b/src/python/WMCore/Cache/GenericDataCache.py
index 723b0cb27..6f3a92945 100644
--- a/src/python/WMCore/Cache/GenericDataCache.py
+++ b/src/python/WMCore/Cache/GenericDataCache.py
@@ -83,6 +83,7 @@ class GenericDataCache(object):
         elif not isinstance(memoryCache, MemoryCacheStruct):
             raise CacheWithWrongStructException(cacheName)
         else:
+            logging.info("Creating generic cache named: %s", cacheName)
             GenericDataCache._dataCache[cacheName] = memoryCache
 
     @staticmethod
diff --git a/src/python/WMCore/ReqMgr/Utils/Validation.py b/src/python/WMCore/ReqMgr/Utils/Validation.py
index 776d6dce8..75873ec3a 100644
--- a/src/python/WMCore/ReqMgr/Utils/Validation.py
+++ b/src/python/WMCore/ReqMgr/Utils/Validation.py
@@ -3,8 +3,7 @@ ReqMgr request handling.
 
 """
 from __future__ import print_function
-from retry import retry
-
+from hashlib import md5
 from WMCore.Lexicon import procdataset
 from WMCore.REST.Auth import authz_match
 from WMCore.ReqMgr.Auth import getWritePermission
@@ -12,7 +11,7 @@ from WMCore.ReqMgr.DataStructs.Request import initialize_request_args, initializ
 from WMCore.ReqMgr.DataStructs.RequestError import InvalidStateTransition, InvalidSpecParameterValue
 from WMCore.ReqMgr.DataStructs.RequestStatus import check_allowed_transition, STATES_ALLOW_ONLY_STATE_TRANSITION
 from WMCore.ReqMgr.Tools.cms import releases, architectures, dashboardActivities
-from WMCore.Services.DBS.DBS3Reader import DBS3Reader as DBSReader
+from WMCore.Services.DBS.DBS3Reader import getDataTiers
 from WMCore.WMFactory import WMFactory
 from WMCore.WMSpec.WMWorkload import WMWorkloadHelper
 from WMCore.WMSpec.WMWorkloadTools import loadSpecClassByType, setArgumentsWithDefault
@@ -251,15 +250,6 @@ def validateOutputDatasets(outDsets, dbsUrl):
     _validateDatatier(datatier, dbsUrl)
 
 
-@retry(tries=5, delay=1)
-def _listDatatiers(dbsUrl):
-    """
-    Query DBS for list of data tiers.
-    If exception is raised, retry up to 5 times with a 1 second delay.
-    """
-    dbsTiers = DBSReader.listDatatiers(dbsUrl)
-    return dbsTiers
-
 def _validateDatatier(datatier, dbsUrl, expiration=3600):
     """
     _validateDatatier_
@@ -267,12 +257,12 @@ def _validateDatatier(datatier, dbsUrl, expiration=3600):
     Provided a list of datatiers extracted from the outputDatasets, checks
     whether they all exist in DBS.
     """
-    cacheName = "dataTierList"
+    cacheName = "dataTierList_" + md5(dbsUrl).hexdigest()
     if not GenericDataCache.cacheExists(cacheName):
-        mc = MemoryCacheStruct(expiration, _listDatatiers, kwargs={'dbsUrl': dbsUrl})
+        mc = MemoryCacheStruct(expiration, getDataTiers, kwargs={'dbsUrl': dbsUrl})
         GenericDataCache.registerCache(cacheName, mc)
 
-    cacheData = GenericDataCache.getCacheData('dataTierList')
+    cacheData = GenericDataCache.getCacheData(cacheName)
     dbsTiers = cacheData.getData()
     badTiers = list(set(datatier) - set(dbsTiers))
     if badTiers:
diff --git a/src/python/WMCore/Services/DBS/DBS3Reader.py b/src/python/WMCore/Services/DBS/DBS3Reader.py
index 1a3c71635..8be8c06cd 100644
--- a/src/python/WMCore/Services/DBS/DBS3Reader.py
+++ b/src/python/WMCore/Services/DBS/DBS3Reader.py
@@ -8,9 +8,9 @@ Readonly DBS Interface
 from __future__ import print_function, division
 
 import logging
-import time
 import traceback
 from collections import defaultdict
+from retry import retry
 
 from RestClient.ErrorHandling.RestClientExceptions import HTTPError
 from dbs.apis.dbsClient import DbsApi
@@ -46,6 +46,18 @@ def remapDBS3Keys(data, stringify=False, **others):
     return data
 
 
+@retry(tries=3, delay=1)
+def getDataTiers(dbsUrl):
+    """
+    Function to retrieve all the datatiers from DBS.
+    NOTE: to be used with some caching (MemoryCacheStruct)
+    :param dbsUrl: the DBS URL string
+    :return: a list of strings/datatiers
+    """
+    dbs = DbsApi(dbsUrl)
+    return [tier['data_tier_name'] for tier in dbs.listDataTiers()]
+
+
 # emulator hook is used to swap the class instance
 # when emulator values are set.
 # Look WMQuality.Emulators.EmulatorSetup module for the values
@@ -56,8 +68,6 @@ class DBS3Reader(object):
 
     General API for reading data from DBS
     """
-    # cache all the datatiers known by DBS
-    _datatiers = {}
 
     def __init__(self, url, logger=None, **contact):
 
@@ -260,40 +270,13 @@ class DBS3Reader(object):
         """
         return [x['logical_file_name'] for x in self.dbs.listFileArray(dataset=datasetPath)]
 
-    @staticmethod
-    def listDatatiers(dbsUrl=None):
+    def listDatatiers(self):
         """
         _listDatatiers_
 
         Get a list of datatiers known by DBS.
         """
-        if dbsUrl is None:
-            msg = "Error in DBSReader.listDatatiers(). DBS Url not set."
-            raise DBSReaderError(msg)
-
-        timenow = int(time.time())
-        if DBS3Reader._datatiers and timenow - 7200 < DBS3Reader._datatiers['ts']:
-            return DBS3Reader._datatiers['tiers']
-
-        try:
-            DBS3Reader._setDatatiersCache(timenow, dbsUrl)
-        except Exception as ex:
-            if not DBS3Reader._datatiers:
-                msg = "Error in DBSReader.listDatatiers\\n%s" % formatEx3(ex)
-                raise DBSReaderError(msg)
-        return DBS3Reader._datatiers['tiers']
-
-    @staticmethod
-    def _setDatatiersCache(ts, dbsUrl):
-        """
-        Set a timestamp and update the list of datatiers cached in
-        the class property
-        """
-        dbs = DbsApi(dbsUrl)
-        DBS3Reader._datatiers['ts'] = ts
-        DBS3Reader._datatiers['tiers'] = [tier['data_tier_name'] for tier in dbs.listDataTiers()]
-
-        return
+        return [tier['data_tier_name'] for tier in self.dbs.listDataTiers()]
 
     def listDatasetFileDetails(self, datasetPath, getParents=False, getLumis=True, validFileOnly=1):
         """
diff --git a/test/python/WMCore_t/Services_t/DBS_t/DBSReader_t.py b/test/python/WMCore_t/Services_t/DBS_t/DBSReader_t.py
index 1ad5dc03e..c094c7a36 100644
--- a/test/python/WMCore_t/Services_t/DBS_t/DBSReader_t.py
+++ b/test/python/WMCore_t/Services_t/DBS_t/DBSReader_t.py
@@ -7,7 +7,7 @@ Unit test for the DBS helper class.
 
 import unittest
 
-from WMCore.Services.DBS.DBS3Reader import DBS3Reader as DBSReader
+from WMCore.Services.DBS.DBS3Reader import getDataTiers, DBS3Reader as DBSReader
 from WMCore.Services.DBS.DBSErrors import DBSReaderError
 from WMQuality.Emulators.EmulatedUnitTestCase import EmulatedUnitTestCase
 
@@ -51,14 +51,26 @@ class DBSReaderTest(EmulatedUnitTestCase):
         """
         listDatatiers returns all datatiers available
         """
-        results = DBSReader.listDatatiers(self.endpoint)
+        self.dbs = DBSReader(self.endpoint)
+        results = self.dbs.listDatatiers()
+        self.assertTrue('RAW' in results)
+        self.assertTrue('GEN-SIM-RECO' in results)
+        self.assertTrue('GEN-SIM' in results)
+        self.assertFalse('RAW-ALAN' in results)
+        return
+
+    def testGetDataTiers(self):
+        """
+        Test the getDataTiers function
+        """
+        results = getDataTiers(self.endpoint)
         self.assertTrue('RAW' in results)
         self.assertTrue('GEN-SIM-RECO' in results)
         self.assertTrue('GEN-SIM' in results)
         self.assertFalse('RAW-ALAN' in results)
         # dbsUrl is mandatory
-        with self.assertRaises(DBSReaderError):
-            _ = DBSReader.listDatatiers()
+        with self.assertRaises(TypeError):
+            _ = getDataTiers()
         return
 
     def testListPrimaryDatasets(self):
@@ -142,9 +154,12 @@ class DBSReaderTest(EmulatedUnitTestCase):
         self.assertTrue(173658 in details[TESTFILE]['Lumis'])
         self.assertEqual(sorted(details[TESTFILE]['Lumis'][173658]),
                          [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
-                          27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
-                          51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74,
-                          75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98,
+                          27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
+                          50,
+                          51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
+                          74,
+                          75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97,
+                          98,
                           99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111])
 
     def testGetDBSSummaryInfo(self):
@@ -230,15 +245,15 @@ class DBSReaderTest(EmulatedUnitTestCase):
         """listFilesInBlockWithParents gets files with parents for a block"""
         self.dbs = DBSReader(self.endpoint)
         files = self.dbs.listFilesInBlockWithParents(
-            '/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0')
+                '/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0')
         self.assertEqual(4, len(files))
         self.assertEqual('/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0',
                          files[0]['block_name'])
         self.assertEqual('/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0',
                          files[0]['BlockName'])
         self.assertEqual(
-            '/store/data/Commissioning2015/Cosmics/RAW/v1/000/238/545/00000/1043E89F-2DCF-E411-9CAE-02163E013751.root',
-            files[0]['ParentList'][0]['LogicalFileName'])
+                '/store/data/Commissioning2015/Cosmics/RAW/v1/000/238/545/00000/1043E89F-2DCF-E411-9CAE-02163E013751.root',
+                files[0]['ParentList'][0]['LogicalFileName'])
 
         self.assertRaises(DBSReaderError, self.dbs.listFilesInBlockWithParents, BLOCK + 'asas')