Repository: dmwm/wmcore
Issue: Use SiteDB v2 for cms site lists in ReqMgr

Currently it still uses SiteDB API v1, this causes bugs such as T1_UK_RAL not appearing on the list for assignment.


Changed files:
src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrConfiguration.py, src/python/WMCore/HTTPFrontEnd/RequestManager/Assign.py, src/python/WMCore/HTTPFrontEnd/RequestManager/Sites.py, src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrWebTools.py

diff --git a/src/python/WMCore/HTTPFrontEnd/RequestManager/Assign.py b/src/python/WMCore/HTTPFrontEnd/RequestManager/Assign.py
index c618d150f..b70bba26b 100644
--- a/src/python/WMCore/HTTPFrontEnd/RequestManager/Assign.py
+++ b/src/python/WMCore/HTTPFrontEnd/RequestManager/Assign.py
@@ -41,7 +41,11 @@ class Assign(WebAPI):
         self.configDBName = config.configDBName
         self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName)
         if not noSiteDB:
-            self.sites = Utilities.sites(config.sitedb)
+            try:
+                self.sites = Utilities.sites(config.sitedb)
+            except Exception, ex:
+                logging.error("Could not retrieve sites from SiteDB, reason: %s" % ex)
+                self.sites = []
         else:
             self.sites = []
         # yet 0.9.40 had also another self.mergedLFNBases which was differentiating
diff --git a/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrConfiguration.py b/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrConfiguration.py
index 68c312516..356fbfd5a 100644
--- a/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrConfiguration.py
+++ b/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrConfiguration.py
@@ -16,7 +16,10 @@ def reqMgrConfig(
     reqMgrHost = "http://%s:%d" % (socket.gethostname().lower(), 8240),
     proxyBase = None,
     couchurl = os.getenv("COUCHURL"),
-    sitedb = 'https://cmsweb.cern.ch/sitedb/json/index/CEtoCMSName?name',
+    # SiteDB v1 API
+    #sitedb = 'https://cmsweb.cern.ch/sitedb/json/index/CEtoCMSName?name',
+    # SiteDB v2 API
+    sitedb = "https://cmsweb.cern.ch/sitedb/data/prod/site-names",
     yuiroot = "/reqmgr/yuiserver/yui",
     configCouchDB = 'reqmgr_config_cache',
     workloadCouchDB = 'reqmgr_workload_cache',
diff --git a/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrWebTools.py b/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrWebTools.py
index b13bebfdb..bd8d8de5d 100644
--- a/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrWebTools.py
+++ b/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrWebTools.py
@@ -4,6 +4,7 @@ import time
 import logging
 import re
 import WMCore.Wrappers.JsonWrapper as JsonWrapper
+from WMCore.Services.Requests import JSONRequests
 import cherrypy
 from os import path
 from xml.dom.minidom import parse as parseDOM
@@ -383,16 +384,38 @@ def priorityMenu(request):
     return ' %i &nbsp<input type="text" size=2 name="%s:priority" />' % (
             request['RequestPriority'],
             request['RequestName'])
+    
 
 def sites(siteDbUrl):
-    """ download a list of all the sites from siteDB """
-    data = JsonWrapper.loads(urllib.urlopen(siteDbUrl).read().replace("'", '"'))
-    # kill duplicates, then put in alphabetical order
-    siteset = set([d['name'] for d in data.values()])
-    # warning: alliteration
-    sitelist = list(siteset)
-    sitelist.sort()
-    return sitelist
+    """
+    Download a list of all the sites from SiteDB.
+    Uses SiteDB v2 API.
+    Using plain urllib gives an error:
+    ...
+        content = zlib.decompress(content)
+        zlib.error: Error -3 while decompressing data: incorrect header check
+    
+    have to go via pycurl.
+    
+    SiteDB returns result in the following form:
+        {'result': [['cms', 'ASGC', 'T1_TW_ASGC'], 
+                    ['cms', 'BY-NCPHEP', 'T3_BY_NCPHEP'],
+                    ... <sites> ...
+                    ['phedex', 'cinvestav', 'T3_MX_Cinvestav']],
+         'desc': {'columns': ['type', 'site_name', 'alias']}}
+    Process the list and select only type:'cms' containing items (aliases).
+    
+    """
+    uriSepIndex = siteDbUrl.find('/', len("https://"))
+    url = siteDbUrl[0:uriSepIndex]
+    uri =  siteDbUrl[uriSepIndex:]
+    caller = JSONRequests(url=url, idict={"pycurl": True})
+    result, status, reason, cached = caller.makeRequest(uri=uri)
+    # iterate over and select 'cms' items
+    sites = [s[2] for s in result["result"] if s[0] == "cms"]
+    sites.sort()
+    return sites
+    
 
 def quote(data):
     """
diff --git a/src/python/WMCore/HTTPFrontEnd/RequestManager/Sites.py b/src/python/WMCore/HTTPFrontEnd/RequestManager/Sites.py
deleted file mode 100644
index 5a9c401f6..000000000
--- a/src/python/WMCore/HTTPFrontEnd/RequestManager/Sites.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import WMCore.Wrappers.JsonWrapper as JsonWrapper
-import urllib
-
-def sites():
-    # download all the sites from siteDB
-    url = 'https://cmsweb.cern.ch/sitedb/json/index/CEtoCMSName?name'
-    data = JsonWrapper.loads(urllib.urlopen(url).read().replace("'", '"'))
-    # kill duplicates, then put in alphabetical order
-    siteset = set([d['name'] for d in data.values()])
-    # warning: alliteration
-    sitelist = list(siteset)
-    sitelist.sort()
-    return sitelist