Repository: dmwm/wmcore
Issue: DBS3Upload crashes with HTTP Error 503

**Impact of the bug**
WMAgent

**Describe the bug**
DBS3Upload crashes if it receives a response like:
```
<center><h1>503 Service Temporarily Unavailable</h1></center>
```
while talking to CMSWEB services, in this case retrieving the parentage chain from wmstatsserver.

**How to reproduce it**
not sure

**Expected behavior**
Catch this specific error message:
```
503 Service Temporarily Unavailable
```
when accessing wmstatsserver and simply skip that components cycle, which will be retried again a few minutes later.

**Additional context and error message**
Similar to issue: https://github.com/dmwm/WMCore/issues/9123

I don't remember seeing this problem with the production cluster, but it happens quite often with the CMS@Home agent. Traceback is:
```
2020-10-17 17:53:41,216:139720193722112:INFO:DBSUploadPoller:Starting the DBSUpload Polling Cycle
2020-10-17 17:53:41,265:139720193722112:WARNING:Service:The cachefile /data/srv/wmagent/v1.4.1.patch1/install/wmagent/DBS3Upload/.wmcore_cache/.wmcore_cache_31961/requests/cmsweb-testbed.cern.ch:8443/15744094587142094_GET_filtered_requ
ests?RequestType=StepChain&ParentageResolved=False&mask=ChainParentageMap does not exist and the service at https://cmsweb-testbed.cern.ch:8443/wmstatsserver/data/filtered_requests?RequestType=StepChain&ParentageResolved=False&mask=Cha
inParentageMap is unavailable - it returned 503 because Service Temporarily Unavailable with result: <html>
<head><title>503 Service Temporarily Unavailable</title></head>
<body>
<center><h1>503 Service Temporarily Unavailable</h1></center>
<hr><center>nginx/1.17.10</center>
</body>
</html>


2020-10-17 17:53:41,349:139720193722112:ERROR:BaseWorkerThread:Error in worker algorithm (1):
Backtrace:
  <WMComponent.DBS3Buffer.DBSUploadPoller.DBSUploadPoller object at 0x7f132bcc12d0> <@========== WMException Start ==========@>
Exception Class: DBSUploadException
Message: Unknown failure while fetching parentage map from WMStats. Error: url=https://cmsweb-testbed.cern.ch:8443/wmstatsserver/data/filtered_requests?RequestType=StepChain&ParentageResolved=False&mask=ChainParentageMap, code=503, rea
son=Service Temporarily Unavailable, headers={'Content-Length': '198', 'Server': 'nginx/1.17.10', 'Connection': 'close', 'Date': 'Sat, 17 Oct 2020 15:53:41 GMT', 'Content-Type': 'text/html', 'CMS-Server-Time': 'D=3087 t=160295002126097
6'}
        ModuleName : WMComponent.DBS3Buffer.DBSUploadPoller
        MethodName : updateDatasetParentageCache
        ClassInstance : None
        FileName : /data/srv/wmagent/v1.4.1.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.4.1.patch1/lib/python2.7/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py
        ClassName : None
        LineNumber : 357
        ErrorNr : 0

Traceback: 
  File "/data/srv/wmagent/v1.4.1.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.4.1.patch1/lib/python2.7/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py", line 345, in updateDatasetParentageCache
    self.datasetParentageCache = self.wmstatsServerSvc.getChildParentDatasetMap()

  File "/data/srv/wmagent/v1.4.1.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.4.1.patch1/lib/python2.7/site-packages/WMCore/Services/WMStatsServer/WMStatsServer.py", line 105, in getChildParentDatasetMap
    results = self.getFilteredActiveData(filter, mask)

  File "/data/srv/wmagent/v1.4.1.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.4.1.patch1/lib/python2.7/site-packages/WMCore/Services/WMStatsServer/WMStatsServer.py", line 92, in getFilteredActiveData
    return self._getResult(callname, verb="GET")

  File "/data/srv/wmagent/v1.4.1.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.4.1.patch1/lib/python2.7/site-packages/WMCore/Services/WMStatsServer/WMStatsServer.py", line 43, in _getResult
    verb=verb, contentType=contentType)

  File "/data/srv/wmagent/v1.4.1.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.4.1.patch1/lib/python2.7/site-packages/WMCore/Services/Service.py", line 208, in refreshCache
    self.getData(cachefile, url, inputdata, incoming_headers, encoder, decoder, verb, contentType)

  File "/data/srv/wmagent/v1.4.1.patch1/sw/slc7_amd64_gcc630/cms/wmagent/1.4.1.patch1/lib/python2.7/site-packages/WMCore/Services/Service.py", line 323, in getData
    raise he
```


Changed files:
test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py, src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py

diff --git a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
index c13392c90..f266b7e29 100644
--- a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
+++ b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
@@ -48,38 +48,6 @@ from WMCore.WMException import WMException
 from WMCore.WorkerThreads.BaseWorkerThread import BaseWorkerThread
 
 
-def createConfigForJSON(config):
-    """
-    Turn a config object into a dictionary of dictionaries
-
-    """
-
-    final = {}
-    for sectionName in config.listSections_():
-        section = getattr(config, sectionName)
-        if hasattr(section, 'dictionary_'):
-            # Create a dictionary key for it
-            final[sectionName] = createDictionaryFromConfig(section)
-
-    return final
-
-
-def createDictionaryFromConfig(configSection):
-    """
-    Recursively create dictionaries from config
-
-    """
-
-    final = configSection.dictionary_()
-
-    for key in final.keys():
-        if hasattr(final[key], 'dictionary_'):
-            # Then we can turn it into a dictionary
-            final[key] = createDictionaryFromConfig(final[key])
-
-    return final
-
-
 def uploadWorker(workInput, results, dbsUrl):
     """
     _uploadWorker_
@@ -141,11 +109,35 @@ def uploadWorker(workInput, results, dbsUrl):
     return
 
 
+def isPassiveError(exceptionObj):
+    """
+    This function will parse the exception object and report whether
+    the error message corresponds to a soft or hard error (hard errors
+    are supposed to let the component crash).
+    :param exceptionObj: any exception object
+    :return: True if it's a soft error, False otherwise
+    """
+    passException = True
+    passiveErrorMsg = ['Service Unavailable', 'Service Temporarily Unavailable',
+                       'Proxy Error', 'Error reading from remote server',
+                       'Connection refused', 'timed out', 'Could not resolve',
+                       'OpenSSL SSL_connect: SSL_ERROR_SYSCALL']
+
+    excReason = getattr(exceptionObj, 'reason', '')
+    for passiveMsg in passiveErrorMsg:
+        if passiveMsg in excReason:
+            break
+        elif passiveMsg in str(exceptionObj):
+            break
+    else:
+        passException = False
+    return passException
+
+
 class DBSUploadException(WMException):
     """
     Holds the exception info for
     all the things that will go wrong
-
     """
 
 
@@ -344,19 +336,16 @@ class DBSUploadPoller(BaseWorkerThread):
         try:
             self.datasetParentageCache = self.wmstatsServerSvc.getChildParentDatasetMap()
         except Exception as ex:
-            reason = getattr(ex, 'reason', '')
-            msg = 'Failed to fetch parentage map from WMStats, skipping this cycle. '
-            msg += 'Exception: %s. Reason: %s. ' % (type(ex).__name__, reason)
-            if 'Service Unavailable' in reason or 'Proxy Error' in reason or \\
-                            'Error reading from remote server' in reason:
-                pass
-            elif 'Connection refused' in str(ex) or 'timed out' in str(ex) or 'Could not resolve' in str(ex):
-                msg += 'Error: %s' % str(ex)
+            excReason = getattr(ex, 'reason', '')
+            errorMsg = 'Failed to fetch parentage map from WMStats, skipping this cycle. '
+            errorMsg += 'Exception: {}. Reason: {}. Error: {}. '.format(type(ex).__name__,
+                                                                        excReason, str(ex))
+            if isPassiveError(ex):
+                logging.warning(errorMsg)
             else:
-                msg = "Unknown failure while fetching parentage map from WMStats. Error: %s" % str(ex)
-                raise DBSUploadException(msg)
-            logging.warning(msg)
-            myThread.logdbClient.post("DBS3Upload_parentMap", msg, "warning")
+                errorMsg += 'Hit a terminal exception in DBSUploadPoller.'
+                raise DBSUploadException(errorMsg)
+            myThread.logdbClient.post("DBS3Upload_parentMap", errorMsg, "warning")
             success = False
         else:
             myThread.logdbClient.delete("DBS3Upload_parentMap", "warning", this_thread=True)
@@ -376,7 +365,7 @@ class DBSUploadPoller(BaseWorkerThread):
         # Load them if we don't have them
         blocksToLoad = []
         for block in openBlocks:
-            if block['blockname'] not in self.blockCache.keys():
+            if block['blockname'] not in self.blockCache:
                 blocksToLoad.append(block['blockname'])
 
         # Now load the blocks
diff --git a/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py b/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py
index 696151301..95a25fd34 100644
--- a/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py
+++ b/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py
@@ -20,7 +20,7 @@ from WMComponent.DBS3Buffer.DBSBufferBlock import DBSBufferBlock
 from WMComponent.DBS3Buffer.DBSBufferDataset import DBSBufferDataset
 from WMComponent.DBS3Buffer.DBSBufferFile import DBSBufferFile
 from WMComponent.DBS3Buffer.DBSBufferUtil import DBSBufferUtil
-from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller
+from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller, isPassiveError
 from WMCore.DAOFactory import DAOFactory
 from WMCore.DataStructs.Run import Run
 from WMCore.Services.UUIDLib import makeUUID
@@ -29,6 +29,14 @@ from WMQuality.Emulators.DBSClient.DBS3API import DbsApi as MockDbsApi
 from WMQuality.TestInit import TestInit
 
 
+class MyTestException(Exception):
+
+    def __init__(self, errorMsg=None):
+        super(MyTestException, self).__init__(errorMsg)
+        if errorMsg is not None:
+            self.reason = errorMsg
+
+
 class DBSUploadTest(unittest.TestCase):
     """
     TestCase for DBSUpload module
@@ -259,40 +267,40 @@ class DBSUploadTest(unittest.TestCase):
 
         results = self.dbsApi.listFileArray(dataset=datasetName, detail=True)
         for result in results:
-            file = None
-            for file in files:
-                if file["lfn"] == result["logical_file_name"]:
+            fileObj = None
+            for fileObj in files:
+                if fileObj["lfn"] == result["logical_file_name"]:
                     break
             else:
-                file = None
+                fileObj = None
 
-            self.assertTrue(file is not None, "Error: File not found.")
-            self.assertEqual(file["size"], result["file_size"],
+            self.assertTrue(fileObj is not None, "Error: File not found.")
+            self.assertEqual(fileObj["size"], result["file_size"],
                              "Error: File size mismatch.")
-            self.assertEqual(file["events"], result["event_count"],
+            self.assertEqual(fileObj["events"], result["event_count"],
                              "Error: Event count mismatch.")
 
-            dbsParents = self.dbsApi.listFileParents(logical_file_name=file["lfn"])
+            dbsParents = self.dbsApi.listFileParents(logical_file_name=fileObj["lfn"])
             if len(dbsParents) == 0:
-                self.assertEqual(len(file["parents"]), 0,
+                self.assertEqual(len(fileObj["parents"]), 0,
                                  "Error: Missing parents.")
             else:
                 for dbsParent in dbsParents[0]["parent_logical_file_name"]:
                     fileParent = None
-                    for fileParent in file["parents"]:
+                    for fileParent in fileObj["parents"]:
                         if fileParent["lfn"] == dbsParent:
                             break
                     else:
                         fileParent = None
 
                     self.assertTrue(fileParent is not None, "Error: Missing parent.")
-                self.assertEqual(len(file["parents"]), len(dbsParents[0]["parent_logical_file_name"]),
+                self.assertEqual(len(fileObj["parents"]), len(dbsParents[0]["parent_logical_file_name"]),
                                  "Error: Wrong number of parents.")
 
-            runLumis = self.dbsApi.listFileLumiArray(logical_file_name=file["lfn"])
+            runLumis = self.dbsApi.listFileLumiArray(logical_file_name=fileObj["lfn"])
             for runLumi in runLumis:
                 fileRun = None
-                for fileRun in file["runs"]:
+                for fileRun in fileObj["runs"]:
                     if fileRun.run == runLumi["run_num"]:
                         for lumi in runLumi["lumi_section_num"]:
                             self.assertTrue(lumi in fileRun.lumis,
@@ -408,9 +416,9 @@ class DBSUploadTest(unittest.TestCase):
             dbsBlock.status = "Open"
             dbsBlock.setDataset(parentFiles[0]["datasetPath"], 'data', 'VALID')
             dbsUtil.createBlocks([dbsBlock])
-            for file in allFiles[i * 5: (i * 5) + 5]:
-                dbsBlock.addFile(file, 'data', 'VALID')
-                dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
+            for fileObj in allFiles[i * 5: (i * 5) + 5]:
+                dbsBlock.addFile(fileObj, 'data', 'VALID')
+                dbsUtil.setBlockFiles({"block": blockName, "filelfn": fileObj["lfn"]})
                 if i < 2:
                     dbsBlock.status = "InDBS"
                 dbsUtil.updateBlocks([dbsBlock])
@@ -425,9 +433,9 @@ class DBSUploadTest(unittest.TestCase):
         dbsBlock.status = "InDBS"
         dbsBlock.setDataset(childFiles[0]["datasetPath"], 'data', 'VALID')
         dbsUtil.createBlocks([dbsBlock])
-        for file in childFiles:
-            dbsBlock.addFile(file, 'data', 'VALID')
-            dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
+        for fileObj in childFiles:
+            dbsBlock.addFile(fileObj, 'data', 'VALID')
+            dbsUtil.setBlockFiles({"block": blockName, "filelfn": fileObj["lfn"]})
 
         dbsUtil.updateFileStatus([dbsBlock], "InDBS")
 
@@ -613,6 +621,21 @@ class DBSUploadTest(unittest.TestCase):
             del os.environ["DONT_TRAP_EXIT"]
         return
 
+    def testPassiveExceptions(self):
+        """
+        Ensure we are properly evaluating passive/hard exceptions in the
+        `isPassiveError` function.
+        """
+        # hard errors
+        for errMessage in ["Unknown exception", "Proxy WRONG Error", None]:
+            self.assertIs(isPassiveError(MyTestException(errMessage)), False)
+        # soft errors
+        for errMessage in ['Service Unavailable', 'Service Temporarily Unavailable',
+                           'Proxy Error', 'Error reading from remote server',
+                           'Connection refused', 'timed out', 'Could not resolve',
+                           'OpenSSL SSL_connect: SSL_ERROR_SYSCALL']:
+            self.assertIs(isPassiveError(MyTestException(errMessage)), True)
+
 
 if __name__ == '__main__':
     unittest.main()