Repository: dmwm/wmcore
Issue: DBS3Upload crashing with a connection time out 

**Impact of the bug**
WMAgent

**Describe the bug**
This week many agents have been going down (DBS3Upload crashes) due to network instabilities at CERN/CMSWEB. This happens when DBS3Upload tries to fetch the dataset parentage map from WMStats.

**How to reproduce it**
I guess making central services unavailable ..

**Expected behavior**
In short:
* the component should not crash, instead it just skips the cycle and try again in the next one
* write to the LogDB that there was such problem and the cycle had to be skipped

**Additional context and error message**
Full error message and traceback from the log:
```
2019-10-17 10:00:15,122:139890905900800:INFO:DBSUploadPoller:Starting the DBSUpload Polling Cycle
2019-10-17 10:04:32,105:139890905900800:ERROR:BaseWorkerThread:Error in worker algorithm (1):
Backtrace:
  <WMComponent.DBS3Buffer.DBSUploadPoller.DBSUploadPoller object at 0x7f3aebdf1f90> <@========== WMException Start ==========@>
Exception Class: DBSUploadException
Message: Unknown failure while fetching parentage map from WMStats. Error: (7, 'Failed to connect to cmsweb.cern.ch port 8443: Connection timed out')
        ModuleName : WMComponent.DBS3Buffer.DBSUploadPoller
        MethodName : updateDatasetParentageCache
        ClassInstance : None
        FileName : /data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py
        ClassName : None
        LineNumber : 355
        ErrorNr : 0

Traceback: 
  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py", line 342, in updateDatasetParentageCache
    self.datasetParentageCache = self.wmstatsServerSvc.getChildParentDatasetMap()

  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMCore/Services/WMStatsServer/WMStatsServer.py", line 105, in getChildParentDatasetMap
    results = self.getFilteredActiveData(filter, mask)

  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMCore/Services/WMStatsServer/WMStatsServer.py", line 92, in getFilteredActiveData
    return self._getResult(callname, verb="GET")

  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMCore/Services/WMStatsServer/WMStatsServer.py", line 43, in _getResult
    verb=verb, contentType=contentType)

  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMCore/Services/Service.py", line 205, in refreshCache
    self.getData(cachefile, url, inputdata, incoming_headers, encoder, decoder, verb, contentType)

  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMCore/Services/Service.py", line 282, in getData
    contentType=contentType)

  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMCore/Services/Requests.py", line 153, in makeRequest
    result = self.makeRequest_pycurl(uri, data, verb, incoming_headers, contentType)

  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMCore/Services/Requests.py", line 178, in makeRequest_pycurl
    ckey=ckey, cert=cert, capath=capath, decode=True)

  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMCore/Services/pycurl_manager.py", line 240, in request
    curl.perform()

<@---------- WMException End ----------@>  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMCore/WorkerThreads/BaseWorkerThread.py", line 182, in __call__
    tSpent, results, _ = algorithmWithDBExceptionHandler(parameters)
  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMCore/Database/DBExceptionHandler.py", line 39, in wrapper
    return f(*args, **kwargs)
  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/Utils/Timers.py", line 24, in wrapper
    res = func(*arg, **kw)
  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py", line 311, in algorithm
    if self.updateDatasetParentageCache() is False:
  File "/data/srv/wmagent/v1.2.6.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.2.6.patch1/lib/python2.7/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py", line 355, in updateDatasetParentageCache
    raise DBSUploadException(msg)

2019-10-17 10:04:32,105:139890905900800:INFO:Harness:>>>Terminating worker threads
2019-10-17 10:04:32,105:139890905900800:ERROR:BaseWorkerThread:Error in event loop (2): <WMComponent.DBS3Buffer.DBSUploadPoller.DBSUploadPoller object at 0x7f3aebdf1f90
```


Changed files:
src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py

diff --git a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
index c0f7d1e60..7e00146f3 100644
--- a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
+++ b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
@@ -224,7 +224,7 @@ class DBSUploadPoller(BaseWorkerThread):
 
         Set up the processing pool for work
         """
-        if len(self.pool) > 0:
+        if self.pool:
             # Then something already exists.  Continue
             return
 
@@ -285,13 +285,13 @@ class DBSUploadPoller(BaseWorkerThread):
         self.workResult = None
         return
 
-    def terminate(self, params):
+    def terminate(self, parameters):
         """
         Do one more pass, then terminate
 
         """
         logging.debug("terminating. doing one more pass before we die")
-        self.algorithm(params)
+        self.algorithm(parameters)
 
     @timeFunction
     def algorithm(self, parameters=None):
@@ -343,13 +343,16 @@ class DBSUploadPoller(BaseWorkerThread):
         except Exception as ex:
             reason = getattr(ex, 'reason', '')
             msg = 'Failed to fetch parentage map from WMStats, skipping this cycle. '
-            if 'Service Unavailable' in reason or 'Proxy Error' in reason or\\
+            msg += 'Exception: %s. Reason: %s. ' % (type(ex).__name__, reason)
+            if 'Service Unavailable' in reason or 'Proxy Error' in reason or \\
                             'Error reading from remote server' in reason:
-                msg += 'Error: %s' % reason
+                pass
             elif 'Connection refused' in str(ex):
                 msg += 'Error: %s' % str(ex)
             elif 'The read operation timed out' in str(ex):
                 msg += 'Error: %s' % str(ex)
+            elif 'Connection timed out' in str(ex):
+                msg += 'Error: %s' % str(ex)
             else:
                 msg = "Unknown failure while fetching parentage map from WMStats. Error: %s" % str(ex)
                 raise DBSUploadException(msg)
@@ -458,8 +461,7 @@ class DBSUploadPoller(BaseWorkerThread):
 
                 files = fileDict.get(location)
 
-                if len(files) > 0:
-
+                if files:
                     currentBlock = self.getBlock(files[0], location, True)
                     currentBlock.setAcquisitionEra(era=dspInfo['AcquisitionEra'])
                     currentBlock.setProcessingVer(procVer=dspInfo['ProcessingVer'])
@@ -617,14 +619,14 @@ class DBSUploadPoller(BaseWorkerThread):
                 createInDBSBuffer.append(block)
 
         # First handle new and updated blocks
-        if len(createInDBSBuffer) > 0 or len(updateInDBSBuffer) > 0:
+        if createInDBSBuffer or updateInDBSBuffer:
             try:
                 myThread.transaction.begin()
-                if len(createInDBSBuffer) > 0:
+                if createInDBSBuffer:
                     self.createBlocksDAO.execute(blocks=createInDBSBuffer,
                                                  conn=myThread.transaction.conn,
                                                  transaction=True)
-                if len(updateInDBSBuffer) > 0:
+                if updateInDBSBuffer:
                     self.updateBlocksDAO.execute(blocks=updateInDBSBuffer,
                                                  conn=myThread.transaction.conn,
                                                  transaction=True)
@@ -648,7 +650,7 @@ class DBSUploadPoller(BaseWorkerThread):
             self.blockCache.get(block.getName()).inBuff = True
 
         # Record new file/block associations in DBSBuffer.
-        if len(self.filesToUpdate) > 0:
+        if self.filesToUpdate:
             try:
                 myThread.transaction.begin()
                 self.setBlockFilesDAO.execute(binds=self.filesToUpdate,
@@ -673,12 +675,12 @@ class DBSUploadPoller(BaseWorkerThread):
             return
 
         # Build the pool if it was closed
-        if len(self.pool) == 0:
+        if not self.pool:
             self.setupPool()
 
         # Finally upload blocks to DBS.
         for block in createInDBS:
-            if len(block.files) < 1:
+            if not block.files:
                 # What are we doing?
                 logging.debug("Skipping empty block")
                 continue
@@ -767,7 +769,7 @@ class DBSUploadPoller(BaseWorkerThread):
                 # Continue to the next block
                 # Block will remain in pending status until it is transferred
 
-        if len(loadedBlocks) > 0:
+        if loadedBlocks:
             try:
                 myThread.transaction.begin()
                 self.updateFilesDAO.execute(blocks=loadedBlocks, status="InDBS",
@@ -808,7 +810,7 @@ class DBSUploadPoller(BaseWorkerThread):
             del self.blockCache[name]
 
         # Clean up the pool so we don't have stuff waiting around
-        if len(self.pool) > 0:
+        if self.pool:
             self.close()
 
         # And we're done
@@ -841,7 +843,7 @@ class DBSUploadPoller(BaseWorkerThread):
                 logging.exception(msg)
 
         # Update the status of those blocks that were truly inserted
-        if len(blocksUploaded) > 0:
+        if blocksUploaded:
             try:
                 myThread.transaction.begin()
                 self.updateBlocksDAO.execute(blocks=blocksUploaded,