Repository: dmwm/wmcore
Issue: DBS3Upload crashing while contacting WMStats for parentage

I believe it's something wrong somehow in the testbed deployment procedure, because it's the first time I see this error (actually it's the second, it happened yesterday as well).
It's actually failing here:
https://github.com/dmwm/WMCore/blob/master/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py#L349

as we hit an unexpected error. Traceback is:
```
2019-03-28 10:30:05,930:139878702106368:INFO:DBSUploadPoller:Starting the DBSUpload Polling Cycle
2019-03-28 10:30:05,994:139878702106368:WARNING:Service:The cachefile /data/srv/wmagent/v1.1.20.patch5/install/wmagent/DBS3Upload/.wmcore_cache/.wmcore_cache_31961/requests/cmsweb-testbed.cern.ch:8443/15744094587142094_GET_filtered_r
equests?RequestType=StepChain&ParentageResolved=False&mask=ChainParentageMap does not exist and the service at https://cmsweb-testbed.cern.ch:8443/wmstatsserver/data/filtered_requests?RequestType=StepChain&ParentageResolved=False&mas
k=ChainParentageMap is unavailable - it returned 400 because Bad Request with result: <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>400 Bad Request</title>
</head><body>
<h1>Bad Request</h1>
<p>Your browser sent a request that this server could not understand.<br />
</p>
</body></html>


2019-03-28 10:30:06,088:139878702106368:ERROR:BaseWorkerThread:Error in worker algorithm (1):
Backtrace:
  <WMComponent.DBS3Buffer.DBSUploadPoller.DBSUploadPoller object at 0x7f380c59f110> <@========== WMException Start ==========@>
Exception Class: DBSUploadException
Message: Unknown failure while fetching parentage map from WMStats. Error: 
        ModuleName : WMComponent.DBS3Buffer.DBSUploadPoller
        MethodName : updateDatasetParentageCache
        ClassInstance : None
        FileName : /data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py
        ClassName : None
        LineNumber : 351
        ErrorNr : 0

Traceback: 
  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py", line 340, in updateDatasetParentageCache
    self.datasetParentageCache = self.wmstatsServerSvc.getChildParentDatasetMap()

  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMCore/Services/WMStatsServer/WMStatsServer.py", line 105, in getChildParentDatasetMap
    results = self.getFilteredActiveData(filter, mask)

  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMCore/Services/WMStatsServer/WMStatsServer.py", line 92, in getFilteredActiveData
    return self._getResult(callname, verb="GET")

  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMCore/Services/WMStatsServer/WMStatsServer.py", line 43, in _getResult
    verb=verb, contentType=contentType)

  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMCore/Services/Service.py", line 205, in refreshCache
    self.getData(cachefile, url, inputdata, incoming_headers, encoder, decoder, verb, contentType)

  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMCore/Services/Service.py", line 319, in getData
    raise he

<@---------- WMException End ----------@>  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMCore/WorkerThreads/BaseWorkerThread.py", line 182, in __call__
    tSpent, results, _ = algorithmWithDBExceptionHandler(parameters)
  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMCore/Database/DBExceptionHandler.py", line 35, in wrapper
    return f(*args, **kwargs)
  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/Utils/Timers.py", line 24, in wrapper
    res = func(*arg, **kw)
  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py", line 311, in algorithm
    if self.updateDatasetParentageCache() is False:
  File "/data/srv/wmagent/v1.1.20.patch5/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch5/lib/python2.7/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py", line 351, in updateDatasetParentageCache
    raise DBSUploadException(msg)
```

Changed files:
test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py, src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py

diff --git a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
index 2d297d1a0..96a507c41 100644
--- a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
+++ b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
@@ -128,31 +128,6 @@ def parseDBSException(exBodyString):
         return exBodyString
 
 
-def isPassiveError(exceptionObj):
-    """
-    This function will parse the exception object and report whether
-    the error message corresponds to a soft or hard error (hard errors
-    are supposed to let the component crash).
-    :param exceptionObj: any exception object
-    :return: True if it's a soft error, False otherwise
-    """
-    passException = True
-    passiveErrorMsg = ['Service Unavailable', 'Service Temporarily Unavailable',
-                       'Proxy Error', 'Error reading from remote server',
-                       'Connection refused', 'timed out', 'Could not resolve',
-                       'OpenSSL SSL_connect: SSL_ERROR_SYSCALL']
-
-    excReason = getattr(exceptionObj, 'reason', '')
-    for passiveMsg in passiveErrorMsg:
-        if passiveMsg in excReason:
-            break
-        elif passiveMsg in str(exceptionObj):
-            break
-    else:
-        passException = False
-    return passException
-
-
 class DBSUploadException(WMException):
     """
     Holds the exception info for
@@ -357,17 +332,12 @@ class DBSUploadPoller(BaseWorkerThread):
         try:
             self.datasetParentageCache = self.wmstatsServerSvc.getChildParentDatasetMap()
         except Exception as ex:
+            success = False
             excReason = getattr(ex, 'reason', '')
             errorMsg = 'Failed to fetch parentage map from WMStats, skipping this cycle. '
-            errorMsg += 'Exception: {}. Reason: {}. Error: {}. '.format(type(ex).__name__,
-                                                                        excReason, str(ex))
-            if isPassiveError(ex):
-                logging.warning(errorMsg)
-            else:
-                errorMsg += 'Hit a terminal exception in DBSUploadPoller.'
-                raise DBSUploadException(errorMsg) from None
+            errorMsg += 'Reason: {}. Error: {}. '.format(excReason, str(ex))
+            logging.error(errorMsg)
             myThread.logdbClient.post("DBS3Upload_parentMap", errorMsg, "warning")
-            success = False
         else:
             myThread.logdbClient.delete("DBS3Upload_parentMap", "warning", this_thread=True)
 
diff --git a/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py b/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py
index 67b60010b..7a454ba9b 100644
--- a/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py
+++ b/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py
@@ -22,7 +22,7 @@ from WMComponent.DBS3Buffer.DBSBufferBlock import DBSBufferBlock
 from WMComponent.DBS3Buffer.DBSBufferDataset import DBSBufferDataset
 from WMComponent.DBS3Buffer.DBSBufferFile import DBSBufferFile
 from WMComponent.DBS3Buffer.DBSBufferUtil import DBSBufferUtil
-from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller, isPassiveError, parseDBSException
+from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller, parseDBSException
 from WMCore.DAOFactory import DAOFactory
 from WMCore.DataStructs.Run import Run
 from WMCore.Services.UUIDLib import makeUUID
@@ -623,21 +623,6 @@ class DBSUploadTest(unittest.TestCase):
             del os.environ["DONT_TRAP_EXIT"]
         return
 
-    def testPassiveExceptions(self):
-        """
-        Ensure we are properly evaluating passive/hard exceptions in the
-        `isPassiveError` function.
-        """
-        # hard errors
-        for errMessage in ["Unknown exception", "Proxy WRONG Error", None]:
-            self.assertIs(isPassiveError(MyTestException(errMessage)), False)
-        # soft errors
-        for errMessage in ['Service Unavailable', 'Service Temporarily Unavailable',
-                           'Proxy Error', 'Error reading from remote server',
-                           'Connection refused', 'timed out', 'Could not resolve',
-                           'OpenSSL SSL_connect: SSL_ERROR_SYSCALL']:
-            self.assertIs(isPassiveError(MyTestException(errMessage)), True)
-
     def testParseDBSException(self):
         """
         test parseDBSException function to parse DBS Go-based server exception message