Repository: dmwm/wmcore
Issue: Better handle for DBS3Upload errors due to "Error: concurrency error"

**Impact of the bug**
WMAgent

**Describe the bug**
Real fix has to come with a (minor) redesign on how data gets injected into DBS Server, to be addressed in this GH ticket:
https://github.com/dmwm/WMCore/issues/11106

Meanwhile, we should do something to minimize those horrible errors and tracebacks in the DBS3Upload component [1].

**How to reproduce it**
Concurrently inject multiple blocks for the same dataset.
 
**Expected behavior**
My suggestion is to catch block injections failing with `Error: concurrency error` and log a simple and friendly error message, explaining that the same block will be retried in the next cycle.
In other words, we should stop dumping that traceback and long/nested error messages in the logs for such well known scenarios.

**Additional context and error message**
[1]
```
2022-05-27 14:21:45,736:139988738467584:ERROR:DBSUploadPoller:Error trying to process block /BuToDStarX_ToD0Pi_inclusive_SoftQCD_TuneCP5_13TeV-pythia8-evtgen/RunIISummer20UL18NanoAODv9-Custom_RDStarPU_BParking_106X_upgrade2018_realistic_v
16_L1v1-v1/NANOAODSIM#b55338fc-46d1-4e97-b65f-1d09a9769c1f through DBS. Error: [{"error":{"reason":"DBSError Code:110 Description:DBS DB insert record error Function:dbs.bulkblocks.insertFilesViaChunks Message: Error: concurrency error","
message":"7cf3dee6307fdaa1f72d0dc03551fd0e6665f32114d752c37819c238cef7a231 unable to insert files, error DBSError Code:110 Description:DBS DB insert record error Function:dbs.bulkblocks.insertFilesViaChunks Message: Error: concurrency err
or","function":"dbs.bulkblocks.InsertBulkBlocksConcurrently","code":110},"http":{"method":"POST","code":400,"timestamp":"2022-05-27 19:21:45.67710595 +0000 UTC m=+191328.489143316","path":"/dbs/prod/global/DBSWriter/bulkblocks","user_agen
t":"DBSClient/Unknown/","x_forwarded_host":"dbs-prod.cern.ch","x_forwarded_for":"137.138.157.32","remote_addr":"137.138.63.204:39950"},"exception":400,"type":"HTTPError","message":"DBSError Code:110 Description:DBS DB insert record error 
Function:dbs.bulkblocks.InsertBulkBlocksConcurrently Message:7cf3dee6307fdaa1f72d0dc03551fd0e6665f32114d752c37819c238cef7a231 unable to insert files, error DBSError Code:110 Description:DBS DB insert record error Function:dbs.bulkblocks.i
nsertFilesViaChunks Message: Error: concurrency error Error: nested DBSError Code:110 Description:DBS DB insert record error Function:dbs.bulkblocks.insertFilesViaChunks Message: Error: concurrency error"}]
Traceback (most recent call last):
  File "/data/srv/wmagent/v2.0.2.patch1/sw/slc7_amd64_gcc630/cms/wmagentpy3/2.0.2.patch1/lib/python3.8/site-packages/WMComponent/DBS3Buffer/DBSUploadPoller.py", line 92, in uploadWorker
    dbsApi.insertBulkBlock(blockDump=block)
  File "/data/srv/wmagent/v2.0.2.patch1/sw/slc7_amd64_gcc630/cms/py3-dbs3-client/4.0.7/lib/python3.8/site-packages/dbs/apis/dbsClient.py", line 606, in insertBulkBlock
    result =  self.__callServer("bulkblocks", data=blockDump, callmethod='POST' )
  File "/data/srv/wmagent/v2.0.2.patch1/sw/slc7_amd64_gcc630/cms/py3-dbs3-client/4.0.7/lib/python3.8/site-packages/dbs/apis/dbsClient.py", line 448, in __callServer
    self.__parseForException(http_error)
  File "/data/srv/wmagent/v2.0.2.patch1/sw/slc7_amd64_gcc630/cms/py3-dbs3-client/4.0.7/lib/python3.8/site-packages/dbs/apis/dbsClient.py", line 492, in __parseForException
    raise http_error
  File "/data/srv/wmagent/v2.0.2.patch1/sw/slc7_amd64_gcc630/cms/py3-dbs3-client/4.0.7/lib/python3.8/site-packages/dbs/apis/dbsClient.py", line 445, in __callServer
    self.http_response = method_func(self.url, method, params, data, request_headers)
  File "/data/srv/wmagent/v2.0.2.patch1/sw/slc7_amd64_gcc630/cms/py3-dbs3-pycurl/3.17.7-comp/lib/python3.8/site-packages/RestClient/RestApi.py", line 42, in post
    return http_request(self._curl)
  File "/data/srv/wmagent/v2.0.2.patch1/sw/slc7_amd64_gcc630/cms/py3-dbs3-pycurl/3.17.7-comp/lib/python3.8/site-packages/RestClient/RequestHandling/HTTPRequest.py", line 62, in __call__
    raise HTTPError(effective_url, http_code, http_response.msg, http_response.raw_header, http_response.body)
RestClient.ErrorHandling.RestClientExceptions.HTTPError: HTTP Error 400: Bad Request
```

Changed files:
test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py, src/python/WMQuality/Emulators/DBSClient/DBS3API.py, src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py

diff --git a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
index 70947729f..b0aeaf996 100644
--- a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
+++ b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py
@@ -99,18 +99,13 @@ def uploadWorker(workInput, results, dbsUrl, gzipEncoding=False):
                 logging.warning("Block %s already exists. Marking it as uploaded.", name)
                 logging.debug("Exception: %s", exString)
                 results.put({'name': name, 'success': "uploaded"})
-            elif 'Proxy Error' in exString:
-                # This is probably a successfully insertion that went bad.
-                # Put it on the check list
-                msg = "Got a proxy error for block %s." % name
-                logging.warning(msg)
-                results.put({'name': name, 'success': "check"})
             elif 'Missing data when inserting to dataset_parents' in exString:
                 msg = "Parent dataset is not inserted yet for block %s." % name
                 logging.warning(msg)
                 results.put({'name': name, 'success': "error", 'error': msg})
             else:
-                msg = "Error trying to process block %s through DBS. Error: %s" % (name, exString)
+                reason = parseDBSException(exString)
+                msg = "Error trying to process block %s through DBS. Error: %s" % (name, reason)
                 logging.exception(msg)
                 logging.debug("block info: %s \\n", block)
                 results.put({'name': name, 'success': "error", 'error': msg})
@@ -118,6 +113,21 @@ def uploadWorker(workInput, results, dbsUrl, gzipEncoding=False):
     return
 
 
+def parseDBSException(exBodyString):
+    """
+    parse DBS Go-based server exception
+    :param exBodyString: exception message body string (not exception).
+    The upstream code extract HTTP body from exception object and pass it here.
+    :return: either (parsed) concise exception message or original body string
+    """
+    try:
+        data = json.loads(exBodyString)
+        # dbs2go always return a list
+        return data[0]['error']['reason']
+    except:
+        return exBodyString
+
+
 def isPassiveError(exceptionObj):
     """
     This function will parse the exception object and report whether
diff --git a/src/python/WMQuality/Emulators/DBSClient/DBS3API.py b/src/python/WMQuality/Emulators/DBSClient/DBS3API.py
index d4e082dc5..9c3da981f 100644
--- a/src/python/WMQuality/Emulators/DBSClient/DBS3API.py
+++ b/src/python/WMQuality/Emulators/DBSClient/DBS3API.py
@@ -54,9 +54,11 @@ class DbsApi(object):
             currentInfo.append(blockDump)
             json.dump(currentInfo, outFileHandle)
 
-        randomNumber = random()
-        if randomNumber < 0.2:
-            raise Exception("Proxy Error, this is a mock proxy error.")
+        # VK: once we introduced parseDBSException function we do not need to simulate Proxy Error
+        # see https://github.com/dmwm/WMCore/pull/11176#issuecomment-1158758332
+        # randomNumber = random()
+        # if randomNumber < 0.2:
+        #     raise Exception("Proxy Error, this is a mock proxy error.")
 
         return
 
diff --git a/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py b/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py
index 58fc72242..67b60010b 100644
--- a/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py
+++ b/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py
@@ -22,7 +22,7 @@ from WMComponent.DBS3Buffer.DBSBufferBlock import DBSBufferBlock
 from WMComponent.DBS3Buffer.DBSBufferDataset import DBSBufferDataset
 from WMComponent.DBS3Buffer.DBSBufferFile import DBSBufferFile
 from WMComponent.DBS3Buffer.DBSBufferUtil import DBSBufferUtil
-from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller, isPassiveError
+from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller, isPassiveError, parseDBSException
 from WMCore.DAOFactory import DAOFactory
 from WMCore.DataStructs.Run import Run
 from WMCore.Services.UUIDLib import makeUUID
@@ -638,6 +638,21 @@ class DBSUploadTest(unittest.TestCase):
                            'OpenSSL SSL_connect: SSL_ERROR_SYSCALL']:
             self.assertIs(isPassiveError(MyTestException(errMessage)), True)
 
+    def testParseDBSException(self):
+        """
+        test parseDBSException function to parse DBS Go-based server exception message
+        """
+        exc = """[{"error":{"reason":"DBSError Code:110 Description:DBS DB insert record error Function:dbs.bulkblocks.insertFilesViaChunks Message: Error: concurrency error","message":"7cf3dee6307fdaa1f72d0dc03551fd0e6665f32114d752c37819c238cef7a231 unable to insert files, error DBSError Code:110 Description:DBS DB insert record error Function:dbs.bulkblocks.insertFilesViaChunks Message: Error: concurrency error","function":"dbs.bulkblocks.InsertBulkBlocksConcurrently","code":110},"http":{"method":"POST","code":400,"timestamp":"2022-05-27 19:21:45.67710595 +0000 UTC m=+191328.489143316","path":"/dbs/prod/global/DBSWriter/bulkblocks","user_agent":"DBSClient/Unknown/","x_forwarded_host":"dbs-prod.cern.ch","x_forwarded_for":"137.138.157.32","remote_addr":"137.138.63.204:39950"},"exception":400,"type":"HTTPError","message":"DBSError Code:110 Description:DBS DB insert record error Function:dbs.bulkblocks.InsertBulkBlocksConcurrently Message:7cf3dee6307fdaa1f72d0dc03551fd0e6665f32114d752c37819c238cef7a231 unable to insert files, error DBSError Code:110 Description:DBS DB insert record error Function:dbs.bulkblocks.insertFilesViaChunks Message: Error: concurrency error Error: nested DBSError Code:110 Description:DBS DB insert record error Function:dbs.bulkblocks.insertFilesViaChunks Message: Error: concurrency error"}]"""
+        data = parseDBSException(exc)
+        expect = 'DBSError Code:110 Description:DBS DB insert record error Function:dbs.bulkblocks.insertFilesViaChunks Message: Error: concurrency error'
+        self.assertEqual(data, expect)
+
+        # test another type of exception
+        exc = Exception()
+        data = parseDBSException(exc)
+        self.assertEqual(data, exc)
+        self.assertIs(isinstance(exc, Exception), True)
+
 
 if __name__ == '__main__':
     unittest.main()