Repository: dmwm/wmcore
Issue: MSUnmerged: Broken MongodB related unittests 

**Impact of the bug**
MSUnmerged

**Describe the bug**
Not exactly a bug. Until now we were having a quite unstable setup of MongoDB related unit tests for both `MSUnmerged` and `MSOutput` as well. Since we now know that the `mogomock` library [1] is doing the job of simulating a real database connection quite well [2] we need to fix these [3] tests too.  

[1]
https://github.com/mongomock/mongomock

[2]
https://cmssdt.cern.ch/dmwm-jenkins/view/All/job/DMWM-WMCore-PR-test/12755/artifact/artifacts/PullRequestReport.html

[3]
https://cmssdt.cern.ch/dmwm-jenkins/view/All/job/DMWM-WMCore-PR-test/12754/testReport/junit/MSUnmerged_t/MSUnmergedTest/testCutPath/

https://cmssdt.cern.ch/dmwm-jenkins/view/All/job/DMWM-WMCore-PR-test/12754/testReport/junit/MSUnmerged_t/MSUnmergedTest/testFilterInclExclDirectories/

https://cmssdt.cern.ch/dmwm-jenkins/view/All/job/DMWM-WMCore-PR-test/12754/testReport/junit/MSUnmerged_t/MSUnmergedTest/testFilterExclDirectories/

https://cmssdt.cern.ch/dmwm-jenkins/view/All/job/DMWM-WMCore-PR-test/12754/testReport/junit/MSUnmerged_t/MSUnmergedTest/testFilterInclDirectories/

https://cmssdt.cern.ch/dmwm-jenkins/view/All/job/DMWM-WMCore-PR-test/12754/testReport/junit/MSUnmerged_t/MSUnmergedTest/testPlineUnmerged/



**How to reproduce it**
All MongoDB related unit tests for MSUnmerged are failing due to lack of MongoDB server at the nodes executing the tests.

**Expected behavior**
Have a decently constructed MongoDB emulation so that  all the tests pass.

**Additional context and error message**
N/A

Changed files:
src/python/WMCore/Database/MongoDB.py, test/python/WMCore_t/MicroService_t/MSUnmerged_t/MSUnmerged_t.py, src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py

diff --git a/src/python/WMCore/Database/MongoDB.py b/src/python/WMCore/Database/MongoDB.py
index d666d124d..43f3b4611 100644
--- a/src/python/WMCore/Database/MongoDB.py
+++ b/src/python/WMCore/Database/MongoDB.py
@@ -9,6 +9,7 @@ from __future__ import division, print_function
 from builtins import str, object
 
 from pymongo import MongoClient, errors, IndexModel
+import mongomock
 
 
 class MongoDB(object):
@@ -24,6 +25,7 @@ class MongoDB(object):
                  collections=None,
                  testIndexes=False,
                  logger=None,
+                 mockMongoDB=False,
                  **kwargs):
         """
         :databases:   A database Name to connect to
@@ -41,8 +43,12 @@ class MongoDB(object):
         self.server = server # '127.0.0.1'
         self.port = port # 8230
         self.logger = logger
+        self.mockMongoDB = mockMongoDB
         try:
-            if replicaset:
+            if mockMongoDB:
+                self.client = mongomock.MongoClient()
+                self.logger.info("NOTICE: MongoDB is set to use mongomock, instead of real database.")
+            elif replicaset:
                 self.client = MongoClient(self.server, self.port, replicaset=replicaset, **kwargs )
             else:
                 self.client = MongoClient(self.server, self.port, **kwargs)
@@ -156,7 +162,8 @@ class MongoDB(object):
         """
         try:
             setattr(self, db, self.client[db])
-            self._dbTest(db)
+            if not self.mockMongoDB:
+                self._dbTest(db)
         except errors.ConnectionFailure as ex:
             msg = "Could not connect to MongoDB server for database: %s\\n%s\\n" % (db, str(ex))
             msg += "Giving up Now."
diff --git a/src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py b/src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py
index 32e2caaa5..d9557328e 100644
--- a/src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py
+++ b/src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py
@@ -112,6 +112,7 @@ class MSUnmerged(MSCore):
         self.msConfig.setdefault("mongoDB", 'msUnmergedDB')
         self.msConfig.setdefault("mongoDBRetryCount", 3)
         self.msConfig.setdefault("mongoDBReplicaset", None)
+        self.msConfig.setdefault("mockMongoDB", False)
 
         msUnmergedIndex = IndexModel('name', unique=True)
         msUnmergedDBConfig = {
@@ -121,6 +122,7 @@ class MSUnmerged(MSCore):
             'replicaset': self.msConfig['mongoDBReplicaset'],
             'logger': self.logger,
             'create': True,
+            'mockMongoDB': self.msConfig['mockMongoDB'],
             'collections': [('msUnmergedColl', msUnmergedIndex)]}
 
         mongoClt = MongoDB(**msUnmergedDBConfig)
diff --git a/test/python/WMCore_t/MicroService_t/MSUnmerged_t/MSUnmerged_t.py b/test/python/WMCore_t/MicroService_t/MSUnmerged_t/MSUnmerged_t.py
index 2f784acbb..bb264b948 100644
--- a/test/python/WMCore_t/MicroService_t/MSUnmerged_t/MSUnmerged_t.py
+++ b/test/python/WMCore_t/MicroService_t/MSUnmerged_t/MSUnmerged_t.py
@@ -67,7 +67,8 @@ def getBasicRSEData():
             "/store/unmerged/express/prod/2020/1/12/log8.tar",
             "/store/unmerged/express/prod/2020/1/12/log9.tar",
             "/store/unmerged/alan/prod/2022/1/12/log10.tar"]
-    rse = {"counters": {"dirsToDeleteAll": 0},
+    rse = {"name": "T2_TestRSE",
+           "counters": {"dirsToDeleteAll": 0},
            "dirs": {"allUnmerged": set(),
                     "toDelete": {},
                     "protected": []},
@@ -124,7 +125,8 @@ class MSUnmergedTest(unittest.TestCase):
                          'wmstatsUrl': 'https://cmsweb-testbed.cern.ch/wmstatsserver',
                          'dirFilterIncl': [],
                          'dirFilterExcl': [],
-                         'emulateGfal2': True}
+                         'emulateGfal2': True,
+                         'mockMongoDB': True}
 
         self.creds = {"client_cert": os.getenv("X509_USER_CERT", "Unknown"),
                       "client_key": os.getenv("X509_USER_KEY", "Unknown")}
@@ -140,7 +142,7 @@ class MSUnmergedTest(unittest.TestCase):
         self.msUnmerged = MSUnmerged(self.msConfig)
         self.msUnmerged.rucioConMon = RucioConMonEmul()
         self.msUnmerged.wmstatsSvc = WMStatsServerEmul()
-        self.msUnmerged.resetCounters()
+        self.msUnmerged.resetServiceCounters()
         self.msUnmerged.rucio = Rucio.Rucio(self.msConfig['rucioAccount'],
                                             hostUrl=self.rucioConfigDict['rucio_host'],
                                             authUrl=self.rucioConfigDict['auth_host'],
@@ -157,23 +159,46 @@ class MSUnmergedTest(unittest.TestCase):
         self.msUnmerged.rseConsStats = self.msUnmerged.rucioConMon.getRSEStats()
         self.msUnmerged.protectedLFNs = set(self.msUnmerged.wmstatsSvc.getProtectedLFNs())
         # Emulate the pipeline run while skipping the last purgeRseObj step
-        self.msUnmerged.resetCounters(plineName=pName)
+        self.msUnmerged.resetServiceCounters()
         rse = self.msUnmerged.updateRSETimestamps(rse, start=True, end=False)
         rse = self.msUnmerged.consRecordAge(rse)
         rse = self.msUnmerged.getUnmergedFiles(rse)
         rse = self.msUnmerged.filterUnmergedFiles(rse)
         rse = self.msUnmerged.cleanRSE(rse)
-        rse = self.msUnmerged.updateRSECounters(rse, pName)
+        rse = self.msUnmerged.updateServiceCounters(rse)
         rse = self.msUnmerged.updateRSETimestamps(rse, start=False, end=True)
         # self.msUnmerged.plineUnmerged.run(rse)
-        expectedRSE = {'counters': {'deletedFail': 0,
-                                    'deletedSuccess': 0,
-                                    'dirsToDelete': 6,
-                                    'dirsToDeleteAll': 6,
-                                    'filesToDelete': 21811,
-                                    'totalNumFiles': 11938},
+        expectedRSE = {'name': 'T2_US_Wisconsin',
+                       'pfnPrefix': None,
+                       'isClean': False,
+                       'rucioConMonStatus': None,
+                       'timestamps': {'endTime': mock.ANY,
+                                      'prevEndTime': 0.0,
+                                      'prevStartTime': 0.0,
+                                      'rseConsStatTime': mock.ANY,
+                                      'startTime': mock.ANY},
+                       "counters": {"totalNumFiles": 11938,
+                                    "totalNumDirs": 11,
+                                    "dirsToDelete": 6,
+                                    "filesToDelete": 0,
+                                    "filesDeletedSuccess": 0,
+                                    "filesDeletedFail": 0,
+                                    "dirsDeletedSuccess": 0,
+                                    "dirsDeletedFail": 0,
+                                    "gfalErrors": {}},
+                       'files': {'allUnmerged': mock.ANY,
+                                 'deletedFail': set(),
+                                 'deletedSuccess': set(),
+                                 'protected': {},
+                                 'toDelete': {'/store/unmerged/Phase2HLTTDRSummer20ReRECOMiniAOD/DYToLL_M-50_TuneCP5_14TeV-pythia8/FEVT/FlatPU0To200_pilot_111X_mcRun4_realistic_T15_v1-v2': mock.ANY,
+                                              '/store/unmerged/Run2016G/DoubleEG/MINIAOD/UL2016_MiniAODv2-v1': mock.ANY,
+                                              '/store/unmerged/SAM/testSRM/SAM-cms-lvs-gridftp.hep.wisc.edu': mock.ANY,
+                                              '/store/unmerged/SAM/testSRM/SAM-cms-lvs-gridftp.hep.wisc.edu/lcg-util': mock.ANY,
+                                              '/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu': mock.ANY,
+                                              '/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu/lcg-util': mock.ANY}},
                        'dirs': {'allUnmerged': set(),
-                                'empty': [],
+                                "deletedSuccess": set(),
+                                "deletedFail": set(),
                                 'protected': {'/store/unmerged/RunIIAutumn18FSPremix/PMSSM_set_1_prompt_1_TuneCP2_13TeV-pythia8/AODSIM/GridpackScan_102X_upgrade2018_realistic_v15-v1',
                                               '/store/unmerged/RunIIFall17DRPremix/Suu_Diquark_S4000_chi1160_TuneCP2_13TeV-madgraph-pythia8/AODSIM/PU2017_94X_mc2017_realistic_v11-v1',
                                               '/store/unmerged/RunIIFall17DRPremix/Suu_Diquark_S4000_chi680_TuneCP2_13TeV-madgraph-pythia8/AODSIM/PU2017_94X_mc2017_realistic_v11-v1',
@@ -184,20 +209,8 @@ class MSUnmergedTest(unittest.TestCase):
                                              '/store/unmerged/SAM/testSRM/SAM-cms-lvs-gridftp.hep.wisc.edu',
                                              '/store/unmerged/SAM/testSRM/SAM-cms-lvs-gridftp.hep.wisc.edu/lcg-util',
                                              '/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu',
-                                             '/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu/lcg-util'}},
-                       'files': {'allUnmerged': mock.ANY,
-                                 'deletedFail': [],
-                                 'deletedSuccess': [],
-                                 'protected': {},
-                                 'toDelete': {'/store/unmerged/Phase2HLTTDRSummer20ReRECOMiniAOD/DYToLL_M-50_TuneCP5_14TeV-pythia8/FEVT/FlatPU0To200_pilot_111X_mcRun4_realistic_T15_v1-v2': mock.ANY,
-                                              '/store/unmerged/Run2016G/DoubleEG/MINIAOD/UL2016_MiniAODv2-v1': mock.ANY,
-                                              '/store/unmerged/SAM/testSRM/SAM-cms-lvs-gridftp.hep.wisc.edu': mock.ANY,
-                                              '/store/unmerged/SAM/testSRM/SAM-cms-lvs-gridftp.hep.wisc.edu/lcg-util': mock.ANY,
-                                              '/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu': mock.ANY,
-                                              '/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu/lcg-util': mock.ANY}},
-                       'isClean': False,
-                       'name': 'T2_US_Wisconsin',
-                       'pfnPrefix': None}
+                                             '/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu/lcg-util'}}
+                       }
         self.assertDictEqual(rse, expectedRSE)
 
     def testCutPath(self):
@@ -224,7 +237,7 @@ class MSUnmergedTest(unittest.TestCase):
                                                      "/store/unmerged/express"]
         self.msUnmerged.protectedLFNs = set()
         filterData = self.msUnmerged.filterUnmergedFiles(rseData)
-        self.assertEqual(filterData['counters']['dirsToDeleteAll'], 2)
+        self.assertEqual(filterData['counters']['dirsToDelete'], 2)
         self.assertItemsEqual(viewkeys(filterData['files']['toDelete']), viewkeys(toDeleteDict))
         self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/data/prod/2018/1/12']),
                               toDeleteDict['/store/unmerged/data/prod/2018/1/12'])
@@ -243,7 +256,7 @@ class MSUnmergedTest(unittest.TestCase):
                                                      "/store/unmerged/alan/prod"]
         self.msUnmerged.protectedLFNs = set()
         filterData = self.msUnmerged.filterUnmergedFiles(rseData)
-        self.assertEqual(filterData['counters']['dirsToDeleteAll'], 2)
+        self.assertEqual(filterData['counters']['dirsToDelete'], 2)
         self.assertItemsEqual(viewkeys(filterData['files']['toDelete']), viewkeys(toDeleteDict))
         self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/data/prod/2018/1/12']),
                               toDeleteDict['/store/unmerged/data/prod/2018/1/12'])
@@ -262,7 +275,7 @@ class MSUnmergedTest(unittest.TestCase):
                                                      "/store/unmerged/alan/prod"]
         self.msUnmerged.protectedLFNs = set()
         filterData = self.msUnmerged.filterUnmergedFiles(rseData)
-        self.assertEqual(filterData['counters']['dirsToDeleteAll'], 1)
+        self.assertEqual(filterData['counters']['dirsToDelete'], 1)
         self.assertItemsEqual(viewkeys(filterData['files']['toDelete']), viewkeys(toDeleteDict))
         self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/express/prod/2020/1/12']),
                               toDeleteDict['/store/unmerged/express/prod/2020/1/12'])