Repository: dmwm/wmcore
Issue: Split ProperlyArchived and ForceArchived counters in MSRuleCleaner

**Impact of the bug**
MSRuleCleaner

**Describe the bug**
For some of the workflows in the queue (so far only ACDCs) we are setting the`ForceArchive` flag to True at  the very begining so that they bypass the whole rest logic of the service and go straight to archival. The reason for that is because we consider the ACDC would not have its won set of Rucio rules but would rather benefit from the already created ones for the main request submission. But we do not distinguished between which flag triggered the archival: either `IsClean` or `ForceArchive`. The result is that the those workflows are contributing to the 'properly archived' counters of the service which accumulate bigger number than the 'properly cleaned' counters, which makes the report from the service be ambiguous  and missleading[1].

[1]  
```
2020-11-19 01:48:31,909:INFO:MSRuleCleaner: Workflows cleaned by pipeline: plineMSTrCont: 197
2020-11-19 01:48:31,910:INFO:MSRuleCleaner: Workflows cleaned by pipeline: plineMSTrBlock: 197
2020-11-19 01:48:31,910:INFO:MSRuleCleaner: Workflows cleaned by pipeline: plineAgentCont: 299
2020-11-19 01:48:31,910:INFO:MSRuleCleaner: Workflows cleaned by pipeline: plineAgentBlock: 299
2020-11-19 01:48:31,910:INFO:MSRuleCleaner: Workflows archived: 380
2020-11-19 01:48:31,910:INFO:MSRuleCleaner: 
Number of processed workflows: 380.
Number of properly cleaned workflows: 299.
Number of archived workflows: 380.
```

**How to reproduce it**
Steps to reproduce the behavior:

**Expected behavior**
To account for which flag triggered the archival in two different counters and include them into the report.

**Additional context and error message**
N/A


Changed files:
src/python/WMCore/MicroService/DataStructs/DefaultStructs.py, test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py, src/python/WMCore/MicroService/Unified/MSRuleCleaner.py

diff --git a/src/python/WMCore/MicroService/DataStructs/DefaultStructs.py b/src/python/WMCore/MicroService/DataStructs/DefaultStructs.py
index 68c72950b..4d4b34ac5 100644
--- a/src/python/WMCore/MicroService/DataStructs/DefaultStructs.py
+++ b/src/python/WMCore/MicroService/DataStructs/DefaultStructs.py
@@ -68,7 +68,8 @@ RULECLEANER_REPORT = dict(thread_id="",
                           error="",
                           total_num_requests=0,
                           clean_num_requests=0,
-                          archived_num_requests=0,
+                          normal_archived_num_requests=0,
+                          force_archived_num_requests=0,
                           success_block_rules_deleted=0,
                           failed_block_rules_deleted=0,
                           success_container_rules_deleted=0,
diff --git a/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py b/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py
index 5fed076fd..64fe090df 100644
--- a/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py
+++ b/src/python/WMCore/MicroService/Unified/MSRuleCleaner.py
@@ -107,7 +107,8 @@ class MSRuleCleaner(MSCore):
 
         # Initialization of the 'cleaned' and 'archived' counters:
         self.wfCounters = {'cleaned': {},
-                           'archived': 0}
+                           'archived': {'normalArchived': 0,
+                                        'forceArchived': 0}}
 
     def resetCounters(self):
         """
@@ -115,7 +116,8 @@ class MSRuleCleaner(MSCore):
         """
         for pline in self.cleanuplines:
             self.wfCounters['cleaned'][pline.name] = 0
-        self.wfCounters['archived'] = 0
+        self.wfCounters['archived']['normalArchived'] = 0
+        self.wfCounters['archived']['forceArchived'] = 0
 
     def execute(self, reqStatus):
         """
@@ -144,17 +146,20 @@ class MSRuleCleaner(MSCore):
 
         # Call _execute() and feed the relevant pipeline with the objects popped from requestRecords
         try:
-            totalNumRequests, cleanNumRequests, archivedNumRequests = self._execute(requestRecords)
+            totalNumRequests, cleanNumRequests, normalArchivedNumRequests, forceArchivedNumRequests = self._execute(requestRecords)
             msg = "\\nNumber of processed workflows: %s."
             msg += "\\nNumber of properly cleaned workflows: %s."
-            msg += "\\nNumber of archived workflows: %s."
+            msg += "\\nNumber of normally archived workflows: %s."
+            msg += "\\nNumber of force archived workflows: %s."
             self.logger.info(msg,
                              totalNumRequests,
                              cleanNumRequests,
-                             archivedNumRequests)
+                             normalArchivedNumRequests,
+                             forceArchivedNumRequests)
             self.updateReportDict(summary, "total_num_requests", totalNumRequests)
             self.updateReportDict(summary, "clean_num_requests", cleanNumRequests)
-            self.updateReportDict(summary, "archived_num_requests", archivedNumRequests)
+            self.updateReportDict(summary, "normal_archived_num_requests", normalArchivedNumRequests)
+            self.updateReportDict(summary, "force_archived_num_requests", forceArchivedNumRequests)
         except Exception as ex:
             msg = "Unknown exception while running MSRuleCleaner thread Error: %s"
             self.logger.exception(msg, str(ex))
@@ -195,9 +200,11 @@ class MSRuleCleaner(MSCore):
         for pline in self.cleanuplines:
             msg = "Workflows cleaned by pipeline: %s: %d"
             self.logger.info(msg, pline.name, self.wfCounters['cleaned'][pline.name])
-        archivedNumRequests = self.wfCounters['archived']
-        self.logger.info("Workflows archived: %d", self.wfCounters['archived'])
-        return totalNumRequests, cleanNumRequests, archivedNumRequests
+        normalArchivedNumRequests = self.wfCounters['archived']['normalArchived']
+        forceArchivedNumRequests = self.wfCounters['archived']['forceArchived']
+        self.logger.info("Workflows normally archived: %d", self.wfCounters['archived']['normalArchived'])
+        self.logger.info("Workflows force archived: %d", self.wfCounters['archived']['forceArchived'])
+        return totalNumRequests, cleanNumRequests, normalArchivedNumRequests, forceArchivedNumRequests
 
     def _dispatchWflow(self, wflow):
         """
@@ -268,7 +275,10 @@ class MSRuleCleaner(MSCore):
         # Archive:
         try:
             self.plineArchive.run(wflow)
-            self.wfCounters['archived'] += 1
+            if wflow['ForceArchive']:
+                self.wfCounters['archived']['forceArchived'] += 1
+            else:
+                self.wfCounters['archived']['normalArchived'] += 1
         except MSRuleCleanerArchival as ex:
             msg = "%s: Archival Error: %s. "
             msg += " Will retry again in the next cycle."
diff --git a/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py b/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py
index a45aa8651..2f489cf7d 100644
--- a/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py
+++ b/test/python/WMCore_t/MicroService_t/Unified_t/MSRuleCleaner_t.py
@@ -159,7 +159,7 @@ class MSRuleCleanerTest(unittest.TestCase):
 
     def testRunning(self):
         result = self.msRuleCleaner._execute(self.reqRecords)
-        self.assertEqual(result, (3, 2, 2))
+        self.assertEqual(result, (3, 2, 2, 0))
 
     def testCheckClean(self):
         # NOTE: All of the bellow checks are well visualized at: