Repository: dmwm/wmcore
Issue: Use SiteDB v2 for cms site lists in ReqMgr

Currently it still uses SiteDB API v1, this causes bugs such as T1_UK_RAL not appearing on the list for assignment.


Changed files:
src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrConfiguration.py, src/python/WMCore/HTTPFrontEnd/RequestManager/Assign.py, src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrWebTools.py

diff --git a/src/python/WMCore/HTTPFrontEnd/RequestManager/Assign.py b/src/python/WMCore/HTTPFrontEnd/RequestManager/Assign.py
index b70bba26b..db1a99a4d 100644
--- a/src/python/WMCore/HTTPFrontEnd/RequestManager/Assign.py
+++ b/src/python/WMCore/HTTPFrontEnd/RequestManager/Assign.py
@@ -1,11 +1,12 @@
-#!/usr/bin/env python
 """
 Main Module for browsing and modifying requests
 as done by the dataOps operators to assign requests
 to processing sites.
 
-Handles site whitelist/blacklist info as well
+Handles site whitelist/blacklist info as well.
+
 """
+
 import types
 import copy
 import logging
@@ -20,12 +21,14 @@ from WMCore.HTTPFrontEnd.RequestManager.ReqMgrAuth import ReqMgrAuth
 from WMCore.Database.CMSCouch import Database
 import WMCore.Lexicon
 from WMCore.Wrappers import JsonWrapper
-
 from WMCore.WebTools.WebAPI import WebAPI
+from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
+
+
 
 class Assign(WebAPI):
     """ Used by data ops to assign requests to processing sites"""
-    def __init__(self, config, noSiteDB = False):
+    def __init__(self, config, noSiteDB=False):
         """
         _init_
 
@@ -42,10 +45,14 @@ class Assign(WebAPI):
         self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName)
         if not noSiteDB:
             try:
-                self.sites = Utilities.sites(config.sitedb)
+                # Download a list of all the sites from SiteDB, uses v2 API.
+                sitedb = SiteDBJSON()
+                self.sites = sitedb.getAllCMSNames()    
+                self.sites.sort()
             except Exception, ex:
-                logging.error("Could not retrieve sites from SiteDB, reason: %s" % ex)
-                self.sites = []
+                msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
+                cherrypy.log(msg)
+                raise
         else:
             self.sites = []
         # yet 0.9.40 had also another self.mergedLFNBases which was differentiating
diff --git a/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrConfiguration.py b/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrConfiguration.py
index 356fbfd5a..975cc07bd 100644
--- a/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrConfiguration.py
+++ b/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrConfiguration.py
@@ -16,10 +16,6 @@ def reqMgrConfig(
     reqMgrHost = "http://%s:%d" % (socket.gethostname().lower(), 8240),
     proxyBase = None,
     couchurl = os.getenv("COUCHURL"),
-    # SiteDB v1 API
-    #sitedb = 'https://cmsweb.cern.ch/sitedb/json/index/CEtoCMSName?name',
-    # SiteDB v2 API
-    sitedb = "https://cmsweb.cern.ch/sitedb/data/prod/site-names",
     yuiroot = "/reqmgr/yuiserver/yui",
     configCouchDB = 'reqmgr_config_cache',
     workloadCouchDB = 'reqmgr_workload_cache',
@@ -90,7 +86,6 @@ def reqMgrConfig(
     active.approve.object = 'WMCore.HTTPFrontEnd.RequestManager.Approve'
     active.section_('assign')
     active.assign.object = 'WMCore.HTTPFrontEnd.RequestManager.Assign'
-    active.assign.sitedb = sitedb
     active.section_('closeout')
     active.closeout.object = 'WMCore.HTTPFrontEnd.RequestManager.CloseOut'
     active.section_('announce')
diff --git a/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrWebTools.py b/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrWebTools.py
index bd8d8de5d..d7b6dbf54 100644
--- a/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrWebTools.py
+++ b/src/python/WMCore/HTTPFrontEnd/RequestManager/ReqMgrWebTools.py
@@ -1,25 +1,29 @@
-""" Functions external to the web interface classes"""
+"""
+Functions external to the web interface classes.
+
+"""
+
 import urllib
 import time
 import logging
 import re
-import WMCore.Wrappers.JsonWrapper as JsonWrapper
-from WMCore.Services.Requests import JSONRequests
-import cherrypy
+import cgi
 from os import path
-from xml.dom.minidom import parse as parseDOM
-from xml.parsers.expat import ExpatError
+import cherrypy
 from cherrypy import HTTPError
 from cherrypy.lib.static import serve_file
+
+from xml.dom.minidom import parse as parseDOM
+from xml.parsers.expat import ExpatError
+
 import WMCore.Lexicon
 from WMCore.ACDC.CouchService import CouchService
 from WMCore.Database.CMSCouch import Database
-import cgi
-import WMCore.RequestManager.RequestDB.Settings.RequestStatus             as RequestStatus
-import WMCore.RequestManager.RequestDB.Interface.Request.ChangeState      as ChangeState
-import WMCore.RequestManager.RequestDB.Interface.Request.GetRequest       as GetRequest
-import WMCore.RequestManager.RequestDB.Interface.Admin.ProdManagement     as ProdManagement
-import WMCore.RequestManager.RequestDB.Interface.Request.ListRequests     as ListRequests
+import WMCore.RequestManager.RequestDB.Settings.RequestStatus as RequestStatus
+import WMCore.RequestManager.RequestDB.Interface.Request.ChangeState as ChangeState
+import WMCore.RequestManager.RequestDB.Interface.Request.GetRequest as GetRequest
+import WMCore.RequestManager.RequestDB.Interface.Admin.ProdManagement as ProdManagement
+import WMCore.RequestManager.RequestDB.Interface.Request.ListRequests as ListRequests
 import WMCore.RequestManager.RequestDB.Interface.Admin.SoftwareManagement as SoftwareAdmin
 import WMCore.Services.WorkQueue.WorkQueue as WorkQueue
 import WMCore.RequestManager.RequestMaker.CheckIn as CheckIn
@@ -29,6 +33,7 @@ from WMCore.WMSpec.StdSpecs.StdBase import WMSpecFactoryException
 from WMCore.RequestManager.DataStructs.RequestSchema import RequestSchema
 from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter
 
+
 def addSiteWildcards(wildcardKeys, sites, wildcardSites):
     """
     _addSiteWildcards_
@@ -386,37 +391,6 @@ def priorityMenu(request):
             request['RequestName'])
     
 
-def sites(siteDbUrl):
-    """
-    Download a list of all the sites from SiteDB.
-    Uses SiteDB v2 API.
-    Using plain urllib gives an error:
-    ...
-        content = zlib.decompress(content)
-        zlib.error: Error -3 while decompressing data: incorrect header check
-    
-    have to go via pycurl.
-    
-    SiteDB returns result in the following form:
-        {'result': [['cms', 'ASGC', 'T1_TW_ASGC'], 
-                    ['cms', 'BY-NCPHEP', 'T3_BY_NCPHEP'],
-                    ... <sites> ...
-                    ['phedex', 'cinvestav', 'T3_MX_Cinvestav']],
-         'desc': {'columns': ['type', 'site_name', 'alias']}}
-    Process the list and select only type:'cms' containing items (aliases).
-    
-    """
-    uriSepIndex = siteDbUrl.find('/', len("https://"))
-    url = siteDbUrl[0:uriSepIndex]
-    uri =  siteDbUrl[uriSepIndex:]
-    caller = JSONRequests(url=url, idict={"pycurl": True})
-    result, status, reason, cached = caller.makeRequest(uri=uri)
-    # iterate over and select 'cms' items
-    sites = [s[2] for s in result["result"] if s[0] == "cms"]
-    sites.sort()
-    return sites
-    
-
 def quote(data):
     """
     Sanitize the data using cgi.escape.
@@ -738,4 +712,4 @@ def retrieveResubmissionChildren(requestName, couchUrl, couchDBName):
 def updateRequestStats(requestName, stats, couchURL, couchDBName):
     couchDB = Database(couchDBName, couchURL)
     return couchDB.updateDocument(requestName, 'ReqMgr', 'totalstats',
-                                         fields=stats)
+                                         fields=stats)
\\ No newline at end of file