Repository: dmwm/wmcore
Issue: AgentStatusWatcher silently dying

It happened twice in vocms0250 this morning. The process gets killed by something
Component:AgentStatusWatcher Not Running

and logs become quiet, of course
```
2018-09-03 10:24:33,839:139914991240960:INFO:AgentStatusPoller:WMBS data collected in: 10 secs
2018-09-03 10:24:34,421:139914991240960:INFO:AgentStatusPoller:Local WorkQueue data collected in: 0 secs
2018-09-03 10:24:35,250:139914991240960:INFO:BaseWorkerThread:AgentStatusPoller took 28.950 secs to execute
```
and nothing else

Changed files:
src/python/WMComponent/AgentStatusWatcher/DrainStatusAPI.py, src/python/WMQuality/Emulators/PyCondorAPI/MockPyCondorAPI.py, src/python/WMComponent/AgentStatusWatcher/DrainStatusPoller.py, src/python/WMCore/Services/PyCondor/PyCondorAPI.py

diff --git a/src/python/WMComponent/AgentStatusWatcher/DrainStatusAPI.py b/src/python/WMComponent/AgentStatusWatcher/DrainStatusAPI.py
index afcd1ffd2..ea8e6750b 100644
--- a/src/python/WMComponent/AgentStatusWatcher/DrainStatusAPI.py
+++ b/src/python/WMComponent/AgentStatusWatcher/DrainStatusAPI.py
@@ -21,6 +21,7 @@ class DrainStatusAPI(object):
         self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
         self.dbsUtil = DBSBufferUtil()
         self.condorAPI = PyCondorAPI()
+        self.condorStates = ("Running", "Idle")
 
     def collectDrainInfo(self):
         """
@@ -51,15 +52,13 @@ class DrainStatusAPI(object):
         Check idle and running jobs in Condor
         """
         results = {}
-        queries = [["1", "idle"], ["2", "running"]]
-
-        for query in queries:
-            jobs = self.condorAPI.getCondorJobs("JobStatus=="+query[0], [])
+        jobs = self.condorAPI.getCondorJobsSummary()
+        for state in self.condorStates:
             # if there is an error, report it instead of the length of an empty list
-            if jobs is None:
-                results[query[1]] = "unknown (schedd query error)"
+            if not jobs:
+                results[state.lower()] = None
             else:
-                results[query[1]] = len(jobs)
+                results[state.lower()] = int(jobs[0].get(state))
 
         return results
 
diff --git a/src/python/WMComponent/AgentStatusWatcher/DrainStatusPoller.py b/src/python/WMComponent/AgentStatusWatcher/DrainStatusPoller.py
index 8cf4a047f..1aba873c4 100644
--- a/src/python/WMComponent/AgentStatusWatcher/DrainStatusPoller.py
+++ b/src/python/WMComponent/AgentStatusWatcher/DrainStatusPoller.py
@@ -38,6 +38,7 @@ class DrainStatusPoller(BaseWorkerThread):
         self.validSpeedDrainConfigKeys = ['CondorPriority', 'NoJobRetries', 'EnableAllSites']
         self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
         self.emailAlert = EmailAlert(config.EmailAlert.dictionary_())
+        self.condorStates = ("Running", "Idle")
 
     @timeFunction
     def algorithm(self, parameters):
@@ -55,7 +56,6 @@ class DrainStatusPoller(BaseWorkerThread):
             return
 
         try:
-
             # see if the agent is in drain mode
             if self.agentConfig["UserDrainMode"] or self.agentConfig["AgentDrainMode"]:
                 # check to see if the agent hit any speed drain thresholds
@@ -170,16 +170,17 @@ class DrainStatusPoller(BaseWorkerThread):
         that need updated for speed draining
         """
         enableKeys = []
+        # first, update our summary of condor jobs
+        totalJobs = self.getTotalCondorJobs()
+        if totalJobs is None:
+            msg = "Cannot check speed drain because there was an error fetching job summary from HTCondor."
+            msg += " Will retry again in the next cycle."
+            logging.warning(msg)
+            return []
 
         # get the current speed drain status
         speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
 
-        # get condor jobs
-        jobs = self.condorAPI.getCondorJobs("", [])
-        if jobs is None:
-            logging.warning("There was an error querying the schedd.  Not checking speed drain thresholds.")
-            return []
-
         # loop through the speed drain configuration and make a list of what thresholds have been hit
         for k, v in viewitems(speedDrainConfig):
             # make sure keys in the speed drain config are valid
@@ -187,10 +188,26 @@ class DrainStatusPoller(BaseWorkerThread):
                 # we always want to apply the condor priority change if the threshold is hit
                 if not v['Enabled'] or k == 'CondorPriority':
                     logging.info("Checking speed drain threshold for %s. ", k)
-                    if len(jobs) < v['Threshold']:
+                    if totalJobs < v['Threshold']:
                         logging.info("Agent will update speed drain configuration for %s. ", k)
                         enableKeys.append(k)
             else:
                 logging.warning("Speed drain configuration error for %s.  Please check aux db contents.", k)
 
         return enableKeys
+
+    def getTotalCondorJobs(self):
+        """
+        Retrieve a summary of the jobs in condor and return an absolute number
+        of the jobs in Idle and Running states.
+        :return: returns an integer with the total number of jobs, or None if it failed.
+        """
+        jobs = self.condorAPI.getCondorJobsSummary()
+        if not jobs:
+            return None
+
+        results = 0
+        if jobs:
+            for state in self.condorStates:
+                results += int(jobs[0].get(state))
+        return results
diff --git a/src/python/WMCore/Services/PyCondor/PyCondorAPI.py b/src/python/WMCore/Services/PyCondor/PyCondorAPI.py
index c8f2d74a0..058b4fcf3 100644
--- a/src/python/WMCore/Services/PyCondor/PyCondorAPI.py
+++ b/src/python/WMCore/Services/PyCondor/PyCondorAPI.py
@@ -9,7 +9,6 @@ from __future__ import print_function, division
 from builtins import str, object
 from past.builtins import basestring
 import logging
-
 try:
     # This module has dependency with python binding for condor package (condor)
     import htcondor
@@ -18,34 +17,64 @@ except ImportError:
 
 
 class PyCondorAPI(object):
+    """
+    Some APIs to interact with HTCondor via the HTCondor python bindings.
+    """
     def __init__(self):
         self.schedd = htcondor.Schedd()
         self.coll = htcondor.Collector()
 
-    def getCondorJobs(self, constraint, attr_list):
+    def recreateSchedd(self):
+        """
+        In case our current schedd object is in a "strange" state, we
+        better recreate it.
         """
-        _getCondorJobs_
+        logging.warning("Recreating Schedd instance due to query error...")
+        self.schedd = htcondor.Schedd()
 
-        Given a job/schedd constraint, return a list of jobs attributes
-        or None if the query to condor fails.
+    def getCondorJobsSummary(self):
+        """
+        Retrieves a job summary from the HTCondor Schedd object
+        :return: a list of classads representing the matching jobs, or None if failed
         """
+        ### NOTE: SummaryOnly does not work with xquery method
+        jobs = None  # return None to signalize the query failed
+        queryOpts = htcondor.htcondor.QueryOpts.SummaryOnly
         try:
-            jobs = self.schedd.query(constraint, attr_list)
-            return jobs
+            return self.schedd.query(opts=queryOpts)
         except Exception:
-            # if there is an error, try to recreate the schedd instance
-            logging.info("Recreating Schedd instance due to query error...")
-            self.schedd = htcondor.Schedd()
+            self.recreateSchedd()
+
         try:
-            jobs = self.schedd.query(constraint, attr_list)
+            jobs = self.schedd.query(opts=queryOpts)
         except Exception as ex:
-            jobs = None  # return None to signalize the query failed
-            msg = "Condor failed to fetch schedd constraints for: %s" % constraint
-            msg += "Error message: %s" % str(ex)
-            logging.exception(msg)
-
+            logging.exception("Failed to fetch summary of jobs from Condor Schedd. Error: %s", str(ex))
         return jobs
 
+    def getCondorJobs(self, constraint='true', attrList=None, limit=-1, opts="Default"):
+        """
+        Given a job/schedd constraint, return a list of job classad.
+        :param constraint: the query constraint (str or ExprTree). Defaults to 'true'
+        :param attrList: a list of attribute strings to be returned in the call.
+                         It defaults to all attributes.
+        :param limit: a limit on the number of matches to return. Defaults to -1 (all)
+        :param opts: string with additional flags for the query. Defaults to Default.
+            https://htcondor.readthedocs.io/en/v8_9_7/apis/python-bindings/api/htcondor.html#htcondor.QueryOpts
+        :return: returns an iterator to the job classads
+        """
+        attrList = attrList or []
+        # if option parameter is invalid, default it to the standard behavior
+        opts = getattr(htcondor.htcondor.QueryOpts, opts, "Default")
+        msg = "Querying condor schedd with params: constraint=%s, attrList=%s, limit=%s, opts=%s"
+        logging.info(msg, constraint, attrList, limit, opts)
+        try:
+            return self.schedd.xquery(constraint, attrList, limit, opts=opts)
+        except Exception:
+            self.recreateSchedd()
+
+        # if we hit another exception, let it be raised up in the chain
+        return self.schedd.xquery(constraint, attrList, limit, opts=opts)
+
     def editCondorJobs(self, job_spec, attr, value):
         """
         _editCondorJobs_
@@ -111,9 +140,10 @@ def getScheddParamValue(param):
     Given a schedd parameter, retrieve it's value with htcondor, e.g.:
     MAX_JOBS_RUNNING, MAX_JOBS_PER_OWNER, etc
     """
+    paramResult = None
     if not isinstance(param, basestring):
         logging.error("Parameter %s must be string type", param)
-        return
+        return paramResult
 
     try:
         paramResult = htcondor.param[param]
@@ -121,7 +151,5 @@ def getScheddParamValue(param):
         msg = "Condor failed to fetch schedd parameter: %s" % param
         msg += "Error message: %s" % str(ex)
         logging.exception(msg)
-        # since it has failed, just return None (not sure it's good?!?)
-        paramResult = None
 
     return paramResult
diff --git a/src/python/WMQuality/Emulators/PyCondorAPI/MockPyCondorAPI.py b/src/python/WMQuality/Emulators/PyCondorAPI/MockPyCondorAPI.py
index 2c66d27e4..39b0e1d02 100644
--- a/src/python/WMQuality/Emulators/PyCondorAPI/MockPyCondorAPI.py
+++ b/src/python/WMQuality/Emulators/PyCondorAPI/MockPyCondorAPI.py
@@ -10,7 +10,13 @@ class MockPyCondorAPI(object):
     def __init__(self, *args, **kwargs):
         print("Using MockPyCondorAPI")
 
-    def getCondorJobs(self, constraint, attr_list):
+    def getCondorJobsSummary(self):
+        """
+        Mock a condor query for the job summary
+        """
+        return []
+
+    def getCondorJobs(self, constraint='true', attrList=None, limit=-1, opts="SummaryOnly"):
         """
         Given a job/schedd constraint, return a list of jobs attributes
         or None if the query to condor fails.