Repository: dmwm/wmcore
Issue: Excessive number of calls to reqmgr_auxiliary couch db

**Impact of the bug**
WMAgent/CouchDB

**Describe the bug**
While checking central couchdb logs to investigate some alarms that have been popping up over the last couple of hours (vocms0742), I noticed many many calls to the agent configuration in `reqmgr_auxiliary`, sometimes multiple calls within the same second.

**How to reproduce it**
No clue

**Expected behavior**
We need to first investigate who is making all these calls. I believe it's WMAgent (we have 3 or 4 components that rely on the agent configuration), however it should be one call per cycle, so one call every few minutes (and some places have a cache of many minutes).

We need to make sure that the agent configuration is fetched only once per component cycle. If needed, some caching in the ReqMgrAux wrapper should be created.

**Additional context and error message**
Couchdb logs in vocms0742 should list something like:
```
... GET /reqmgr_auxiliary/WMAGENT_CONFIG_vocms0243.cern.ch 200
```

Changed files:
src/python/WMComponent/JobArchiver/JobArchiverPoller.py, etc/WMAgentConfig.py, src/python/WMComponent/AgentStatusWatcher/DrainStatusPoller.py, src/python/WMComponent/ErrorHandler/ErrorHandlerPoller.py, src/python/WMCore/Services/ReqMgrAux/ReqMgrAux.py, src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py

diff --git a/etc/WMAgentConfig.py b/etc/WMAgentConfig.py
index 52d495b2c..9ab7d0521 100644
--- a/etc/WMAgentConfig.py
+++ b/etc/WMAgentConfig.py
@@ -91,6 +91,8 @@ config.General.logdb_name = logDBName
 config.General.central_logdb_url = "need to get from secrets file"
 config.General.ReqMgr2ServiceURL = "ReqMgr2 rest service"
 config.General.centralWMStatsURL = "Central WMStats URL"
+# ReqMgrAux disk cache duration (in hours), set to 5 minutes: 5 / 60 = 0.083
+config.General.ReqMgrAuxCacheDuration = 0.083
 
 config.section_("JobStateMachine")
 config.JobStateMachine.couchurl = couchURL
diff --git a/src/python/WMComponent/AgentStatusWatcher/DrainStatusPoller.py b/src/python/WMComponent/AgentStatusWatcher/DrainStatusPoller.py
index 1aba873c4..89cfd4e6f 100644
--- a/src/python/WMComponent/AgentStatusWatcher/DrainStatusPoller.py
+++ b/src/python/WMComponent/AgentStatusWatcher/DrainStatusPoller.py
@@ -36,7 +36,8 @@ class DrainStatusPoller(BaseWorkerThread):
         self.agentConfig = {}
         self.previousConfig = {}
         self.validSpeedDrainConfigKeys = ['CondorPriority', 'NoJobRetries', 'EnableAllSites']
-        self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
+        cacheduration = getattr(self.config.General, "ReqMgrAuxCacheDuration", 5 / 60)  # 5 minutes
+        self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL, httpDict={'cacheduration': cacheduration})
         self.emailAlert = EmailAlert(config.EmailAlert.dictionary_())
         self.condorStates = ("Running", "Idle")
 
diff --git a/src/python/WMComponent/ErrorHandler/ErrorHandlerPoller.py b/src/python/WMComponent/ErrorHandler/ErrorHandlerPoller.py
index 1da590d00..853b8f2dd 100644
--- a/src/python/WMComponent/ErrorHandler/ErrorHandlerPoller.py
+++ b/src/python/WMComponent/ErrorHandler/ErrorHandlerPoller.py
@@ -73,7 +73,8 @@ class ErrorHandlerPoller(BaseWorkerThread):
             self.reqAuxDB = None
             self.maxRetries = self.config.ErrorHandler.maxRetries
         else:
-            self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
+            cacheduration = getattr(self.config.General, "ReqMgrAuxCacheDuration", 5 / 60)  # 5 minutes
+            self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL, httpDict={'cacheduration': cacheduration})
 
         self.exitCodesNoRetry = []
         self.maxProcessSize = getattr(self.config.ErrorHandler, 'maxProcessSize', 250)
diff --git a/src/python/WMComponent/JobArchiver/JobArchiverPoller.py b/src/python/WMComponent/JobArchiver/JobArchiverPoller.py
index 55b931150..e95c009ca 100644
--- a/src/python/WMComponent/JobArchiver/JobArchiverPoller.py
+++ b/src/python/WMComponent/JobArchiver/JobArchiverPoller.py
@@ -300,11 +300,14 @@ class JobArchiverPoller(BaseWorkerThread):
         markAction = self.daoFactory(classname="Workflow.MarkInjectedWorkflows")
         result = getAction.execute()
 
+        # Get the drain mode status
+        drainMode = isDrainMode(self.config)
+
         # Check each result to see if it is injected:
         injected = []
         for name in result:
             try:
-                if self.workQueue.getWMBSInjectionStatus(name, isDrainMode(self.config)):
+                if self.workQueue.getWMBSInjectionStatus(name, drainMode):
                     injected.append(name)
             except WorkQueueNoMatchingElements:
                 # workflow not known - free to cleanup
diff --git a/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py b/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py
index 17746921b..0f481aa3e 100644
--- a/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py
+++ b/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py
@@ -137,7 +137,8 @@ class JobSubmitterPoller(BaseWorkerThread):
             # only set up this when reqmgr is used (not Tier0)
             self.reqmgr2Svc = ReqMgr(self.config.General.ReqMgr2ServiceURL)
             self.abortedAndForceCompleteWorkflowCache = self.reqmgr2Svc.getAbortedAndForceCompleteRequestsFromMemoryCache()
-            self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
+            cacheduration = getattr(self.config.General, "ReqMgrAuxCacheDuration", 5 / 60)  # 5 minutes
+            self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL, httpDict={'cacheduration': cacheduration})
         else:
             # Tier0 Case - just for the clarity (This private variable shouldn't be used
             self.abortedAndForceCompleteWorkflowCache = None
diff --git a/src/python/WMCore/Services/ReqMgrAux/ReqMgrAux.py b/src/python/WMCore/Services/ReqMgrAux/ReqMgrAux.py
index dbff84eaa..1256218e0 100644
--- a/src/python/WMCore/Services/ReqMgrAux/ReqMgrAux.py
+++ b/src/python/WMCore/Services/ReqMgrAux/ReqMgrAux.py
@@ -35,7 +35,7 @@ class ReqMgrAux(Service):
         # This is only for the unittest: never set it true unless it is unittest
         self._noStale = False
 
-    def _getResult(self, callname, clearCache=True, args=None, verb="GET",
+    def _getResult(self, callname, clearCache=False, args=None, verb="GET",
                    encoder=json.loads, decoder=json.loads, contentType=None):
         """
         _getResult_