Repository: dmwm/wmcore
Issue: Cache TagCollector data

**Impact of the bug**
ReqMgr2

**Describe the bug**
It looks like we are not using any caching mechanism for the releases XML data (TagCollector) used to validate CMSSW and ScramArchs during request creation (and/or assignment).

**How to reproduce it**
Not easily reproducible

**Expected behavior**
We should either use the Services caching mechanism or MemoryCache and avoid unneeded http queries. Setting the cache to 1h should be enough.

**Additional context and error message**
Traceback from ReqMgr2:
```
[23/Sep/2019:00:39:07]  Updating request "None" with these user-provided args: {u'PrepID': u'task_EXO...
WARNING:root:Http request failed, retrying once again..
WARNING:root:The cachefile <cStringIO.StringO object at 0x7f9725971a08> does not exist and the service at https://cmssdt.cern.ch/SDT/cgi-bin/ReleasesXML/ raised a error(111, 'Connection refused') when accessed
ERROR:root:About to raise exception <@========== WMException Start ==========@>
Exception Class: WMSpecFactoryException
Message: [Errno 111] Connection refused
        ModuleName : WMCore.WMSpec.WMWorkloadTools
        MethodName : _validateArgFunction
        ClassInstance : None
        FileName : /data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/WMSpec/WMWorkloadTools.py
        ClassName : None
        LineNumber : 102
        ErrorNr : 0

Traceback: 
  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/WMSpec/WMWorkloadTools.py", line 93, in _validateArgFunction
    if not valFunction(value):

  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/WMSpec/StdSpecs/StdBase.py", line 1030, in <lambda>
    "ScramArch": {"validate": lambda x: all([y in architectures() for y in x]),

  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/ReqMgr/Tools/cms.py", line 224, in architectures
    return TC.architectures()

  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/Services/TagCollector/TagCollector.py", line 88, in architectures
    for row in self.data():

  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/Services/TagCollector/TagCollector.py", line 67, in data
    data = self._getResult()

  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/Services/TagCollector/TagCollector.py", line 53, in _getResult
    verb=verb, contentType=contentType)

  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/Services/Service.py", line 205, in refreshCache
    self.getData(cachefile, url, inputdata, incoming_headers, encoder, decoder, verb, contentType)

  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/Services/Service.py", line 318, in getData
    raise he

<@---------- WMException End ----------@>
[23/Sep/2019:00:39:07]  Error: Traceback (most recent call last):
  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/ReqMgr/Service/Request.py", line 193, in validate
    self._validateRequestBase(param, safe, validate_request_create_args)
  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/ReqMgr/Service/Request.py", line 88, in _validateRequestBase
    workload, r_args = valFunc(args, self.config, self.reqmgr_db_service, param)
  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/ReqMgr/Utils/Validation.py", line 101, in validate_request_create_args
    request_args)
  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/WMSpec/StdSpecs/StdBase.py", line 915, in factoryWorkloadConstruction
    self.masterValidation(schema=arguments)
  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/WMSpec/StdSpecs/StdBase.py", line 940, in masterValidation
    self.raiseValidationException(str(ex))
  File "/data/srv/beHG1909e/sw/slc7_amd64_gcc630/cms/reqmgr2/1.2.6.pre4/lib/python2.7/site-packages/WMCore/WMSpec/StdSpecs/StdBase.py", line 952, in raiseValidationException
    raise WMSpecFactoryException(message=msg)
WMSpecFactoryException: <@========== WMException Start ==========@>
```

Changed files:
src/python/WMCore/Services/TagCollector/TagCollector.py

diff --git a/src/python/WMCore/Services/TagCollector/TagCollector.py b/src/python/WMCore/Services/TagCollector/TagCollector.py
index 0a45f5feb..45388f92d 100644
--- a/src/python/WMCore/Services/TagCollector/TagCollector.py
+++ b/src/python/WMCore/Services/TagCollector/TagCollector.py
@@ -5,7 +5,7 @@ import logging
 from collections import defaultdict
 from WMCore.Services.Service import Service
 from WMCore.Services.TagCollector.XMLUtils import xml_parser
-
+from urlparse import urlparse
 
 class TagCollector(Service):
     """
@@ -13,23 +13,26 @@ class TagCollector(Service):
     Provides non-deprecated CMSSW releases in all their ScramArchs (not only prod)
     """
 
-    def __init__(self, url=None, logger=None, **kwargs):
+    def __init__(self, url=None, logger=None, configDict=None, **kwargs):
         """
         responseType will be either xml or json
         """
         defaultURL = "https://cmssdt.cern.ch/SDT/cgi-bin/ReleasesXML"
+        url = url or defaultURL
+        parsedUrl = urlparse(url)
+        self.cFileUrlPath = parsedUrl.path.replace("/", "_")
         # all releases types and all their archs
         self.tcArgs = kwargs
         self.tcArgs.setdefault("anytype", 1)
         self.tcArgs.setdefault("anyarch", 1)
 
-        params = {}
-        params["timeout"] = 300
-        params['endpoint'] = url or defaultURL
-        params.setdefault('cacheduration', 1)
-        params['logger'] = logger if logger else logging.getLogger()
-
-        Service.__init__(self, params)
+        configDict = configDict or {}
+        configDict.setdefault('endpoint', url)
+        configDict.setdefault("timeout", 300)
+        configDict.setdefault('cacheduration', 1)
+        configDict['logger'] = logger if logger else logging.getLogger()
+        super(TagCollector, self).__init__(configDict)
+        self['logger'].debug("Initializing TagCollector with url: %s", self['endpoint'])
 
     def _getResult(self, callname="", clearCache=False,
                    args=None, verb="GET", encoder=None, decoder=None,
@@ -45,11 +48,17 @@ class TagCollector(Service):
         if not args:
             args = self.tcArgs
 
-        cfile = callname.replace("/", "_")
+        cFile = '%s_%s'% (self.cFileUrlPath, callname.replace("/", "_"))
+        # If no callname or url path, the base host is getting queried
+        if cFile == '_':
+            cFile = 'baseRequest'
+
         if clearCache:
-            self.clearCache(cfile, args, verb)
+            self.clearCache(cFile, args, verb)
 
-        f = self.refreshCache(cfile, callname, args, encoder=encoder,
+        # Note cFile is just the base name pattern, args 
+        # are also considered for the end filename in the method below
+        f = self.refreshCache(cFile, callname, args, encoder=encoder,
                               verb=verb, contentType=contentType)
         result = f.read()
         f.close()