Repository: dmwm/wmcore
Issue: [MSTransferor] TypeError exception when evaluating STUCK rules

**Impact of the bug**
MSTransferor

**Describe the bug**
This issue popped up today while talking to Rucio Int, and it's not clear to me whether it could show up in production as well.

There is a data type mismatch when evaluating a rule that is in the STUCK state, which could have `stuck_at=None`, thus raising an exception when getting the difference between its creation and the moment it got stuck. See traceback (with an additional debugging put in place) here [1]

**How to reproduce it**
Not sure

**Expected behavior**
There are two items to be fixed here:
1) `stuck_at` could be None. The code needs to handle it (still figuring out the details)
2) we need to properly convert those string date/time to something that we can operate/subtract

**Additional context and error message**
[1]
```
2021-04-29 17:38:20,955:INFO:PycurlRucio: AMR item: {u'locks_ok_cnt': 0, u'source_replica_expression': None, u'weight': None, u'purge_replicas': False, u'rse_expression': u'T1_US_FNAL_Disk', u'updated_at': u'Thu, 29 Apr 2021 15:19:30 UTC', u'meta': None, u'child_rule_id': None, u'id': u'3a4f4e10708c425cb6858855667a43a4', u'ignore_account_limit': False, u'locks_stuck_cnt': 8, u'locks_replicating_cnt': 0, u'notification': u'NO', u'copies': 1, u'comments': u'WMCore MSTransferor input data placement', u'split_container': False, u'priority': 3, u'state': u'STUCK', u'scope': u'cms', u'subscription_id': None, u'ignore_availability': False, u'stuck_at': None, u'error': u'RequestErrMsg.MISMATCH_SCHEME:RequestState.MISMATCH_SCHEME', u'eol_at': None, u'expires_at': u'Fri, 30 Apr 2021 13:15:42 UTC', u'did_type': u'CONTAINER', u'account': u'wmcore_transferor', u'locked': False, u'name': u'/RelValMinBias_14TeV/CMSSW_11_2_0_pre8-112X_mcRun3_2024_realistic_v10_forTrk-v1/GEN-SIM', u'created_at': u'Thu, 29 Apr 2021 13:15:42 UTC', u'activity': u'Production Input', u'grouping': u'ALL'}
2021-04-29 17:38:20,955:ERROR:PycurlRucio: listReplicationRules function did not return a valid response for container: /RelValMinBias_14TeV/CMSSW_11_2_0_pre8-112X_mcRun3_2024_realistic_v10_forTrk-v1/GEN-SIM.Server responded with: unsupported operand type(s) for -: 'NoneType' and 'unicode'
Error: {"id": "3a4f4e10708c425cb6858855667a43a4", "subscription_id": null, "account": "wmcore_transferor", "scope": "cms", "name": "/RelValMinBias_14TeV/CMSSW_11_2_0_pre8-112X_mcRun3_2024_realistic_v10_forTrk-v1/GEN-SIM", "did_type": "CONTAINER", "state": "STUCK", "error": "RequestErrMsg.MISMATCH_SCHEME:RequestState.MISMATCH_SCHEME", "rse_expression": "T1_US_FNAL_Disk", "copies": 1, "expires_at": "Fri, 30 Apr 2021 13:15:42 UTC", "weight": null, "locked": false, "locks_ok_cnt": 0, "locks_replicating_cnt": 0, "locks_stuck_cnt": 8, "source_replica_expression": null, "activity": "Production Input", "grouping": "ALL", "notification": "NO", "stuck_at": null, "purge_replicas": false, "ignore_availability": false, "ignore_account_limit": false, "priority": 3, "comments": "WMCore MSTransferor input data placement", "child_rule_id": null, "eol_at": null, "split_container": false, "meta": null, "created_at": "Thu, 29 Apr 2021 13:15:42 UTC", "updated_at": "Thu, 29 Apr 2021 15:19:30 UTC"}
Traceback (most recent call last):
  File "/data/srv/HG2105c/sw/slc7_amd64_gcc630/cms/reqmgr2ms/0.4.8.pre3/lib/python2.7/site-packages/WMCore/MicroService/Tools/PycurlRucio.py", line 192, in listReplicationRules
    timeDiff = item['stuck_at'] - item['created_at']
TypeError: unsupported operand type(s) for -: 'NoneType' and 'unicode'
```

Changed files:
src/python/WMCore/MicroService/Tools/PycurlRucio.py, test/python/WMCore_t/MicroService_t/Tools_t/PycurlRucio_t.py

diff --git a/src/python/WMCore/MicroService/Tools/PycurlRucio.py b/src/python/WMCore/MicroService/Tools/PycurlRucio.py
index 733eb8399..7756ad4a1 100644
--- a/src/python/WMCore/MicroService/Tools/PycurlRucio.py
+++ b/src/python/WMCore/MicroService/Tools/PycurlRucio.py
@@ -50,6 +50,18 @@ def parseNewLineJson(stream):
             yield json.loads(line)
 
 
+def stringDateToEpoch(strDate):
+    """
+    Given a date/time in the format of:
+        'Thu, 29 Apr 2021 13:15:42 UTC'
+    it returns an integer with the equivalent EPOCH time
+    :param strDate: a string with the date and time
+    :return: the equivalent EPOCH time (integer)
+    """
+    timestamp = datetime.datetime.strptime(strDate, "%a, %d %b %Y %H:%M:%S %Z")
+    return int(timestamp.strftime('%s'))
+
+
 def getRucioToken(rucioAuthUrl, rucioAcct):
     """
     Provided a Rucio account, fetch a token from the authentication server
@@ -69,8 +81,7 @@ def getRucioToken(rucioAuthUrl, rucioAcct):
         tokenExpiration = res.getHeaderKey('X-Rucio-Auth-Token-Expires')
         logging.info("Retrieved Rucio token valid until: %s", tokenExpiration)
         # convert the human readable expiration time to EPOCH time
-        tokenExpiration = datetime.datetime.strptime(tokenExpiration, "%a, %d %b %Y %H:%M:%S %Z")
-        tokenExpiration = int(tokenExpiration.strftime('%s'))
+        tokenExpiration = stringDateToEpoch(tokenExpiration)
         return userToken, tokenExpiration
 
     raise RuntimeError("Failed to acquire a Rucio token. Error: {}".format(res.getReason()))
@@ -188,16 +199,24 @@ def listReplicationRules(containers, rucioAccount, grouping,
                         logging.warning(msg)
                         continue
 
-                    timeDiff = item['stuck_at'] - item['created_at']
-                    if int(timeDiff.days) > STUCK_LIMIT:
+                    # then calculate for how long it's been stuck
+                    utcTimeNow = int(datetime.datetime.utcnow().strftime('%s'))
+                    if item['stuck_at']:
+                        stuckAt = stringDateToEpoch(item['stuck_at'])
+                    else:
+                        # consider it to be stuck since its creation
+                        stuckAt = stringDateToEpoch(item['created_at'])
+
+                    daysStuck = (utcTimeNow - stuckAt) // (24 * 60 * 60)
+                    if daysStuck > STUCK_LIMIT:
                         msg = "Container {} has a STUCK rule for {} days (limit set to: {}).".format(container,
-                                                                                                     timeDiff.days,
+                                                                                                     daysStuck,
                                                                                                      STUCK_LIMIT)
                         msg += " Not going to use it! Rule info: {}".format(item)
                         logging.warning(msg)
                         continue
                     else:
-                        msg = "Container {} has a STUCK rule for only {} days.".format(container, timeDiff.days)
+                        msg = "Container {} has a STUCK rule for only {} days.".format(container, daysStuck)
                         msg += " Considering it for the pileup location"
                         logging.info(msg)
                 else:
diff --git a/test/python/WMCore_t/MicroService_t/Tools_t/PycurlRucio_t.py b/test/python/WMCore_t/MicroService_t/Tools_t/PycurlRucio_t.py
index ec945b1a0..a70e5ed57 100644
--- a/test/python/WMCore_t/MicroService_t/Tools_t/PycurlRucio_t.py
+++ b/test/python/WMCore_t/MicroService_t/Tools_t/PycurlRucio_t.py
@@ -9,7 +9,7 @@ import unittest
 
 from WMCore.MicroService.Tools.PycurlRucio import (getRucioToken, parseNewLineJson,
                                                    getPileupContainerSizesRucio, listReplicationRules,
-                                                   getBlocksAndSizeRucio)
+                                                   getBlocksAndSizeRucio, stringDateToEpoch)
 
 CONT1 = "/Pseudoscalar2HDM_MonoZLL_mScan_mH-500_ma-300/DMWM_Test-TC_PreMix_Agent122_Validation_Alanv1-v11/MINIAODSIM"
 CONT2 = "/RelValH125GGgluonfusion_14/Integ_Test-SC_LumiMask_PhEDEx_HG2004_Val_Privv19-v11/NANOAODSIM"
@@ -26,7 +26,7 @@ class PycurlRucioTests(unittest.TestCase):
         self.rucioAuthUrl = "https://cmsrucio-auth-int.cern.ch"
         self.rucioAccount = "wma_test"
         self.rucioScope = "cms"
-        self.rucioToken, self.tokenValidity = getRucioToken(self.rucioAuthUrl, self.rucioAccount)
+        #self.rucioToken, self.tokenValidity = getRucioToken(self.rucioAuthUrl, self.rucioAccount)
         self.badDID = "/wrong/did/name"
 
     def testParseNewLineJson(self):
@@ -45,6 +45,7 @@ class PycurlRucioTests(unittest.TestCase):
         Test the getPileupContainerSizesRucio function, which fetches the container
         total bytes.
         """
+        self.rucioToken, self.tokenValidity = getRucioToken(self.rucioAuthUrl, self.rucioAccount)
         resp = getPileupContainerSizesRucio([], self.rucioUrl,
                                             self.rucioToken, scope=self.rucioScope)
         self.assertEqual(resp, {})
@@ -62,6 +63,7 @@ class PycurlRucioTests(unittest.TestCase):
         Test the listReplicationRules function, which fetches the container
         rules a return a list of RSEs key'ed by the container name.
         """
+        self.rucioToken, self.tokenValidity = getRucioToken(self.rucioAuthUrl, self.rucioAccount)
         resp = listReplicationRules([], self.rucioAccount, grouping="A",
                                     rucioUrl=self.rucioUrl, rucioToken=self.rucioToken, scope=self.rucioScope)
         self.assertEqual(resp, {})
@@ -83,6 +85,7 @@ class PycurlRucioTests(unittest.TestCase):
         Test the getBlocksAndSizeRucio function, which fetches all the blocks
         (in a container) and their sizes.
         """
+        self.rucioToken, self.tokenValidity = getRucioToken(self.rucioAuthUrl, self.rucioAccount)
         BLOCK = "/DMSimp_MonoZLL_NLO_Vector_TuneCP3_GQ0p25_GDM1p0_MY1-500_MXd-1/RunIIFall17NanoAODv4-PU2017_12Apr2018_Nano14Dec2018_102X_mc2017_realistic_v6-v1/NANOAODSIM#048c25e9-38bb-496d-86f7-405ffd3d3fd8"
         resp = getBlocksAndSizeRucio([], self.rucioUrl, self.rucioToken, self.rucioScope)
         self.assertEqual(resp, {})
@@ -95,6 +98,15 @@ class PycurlRucioTests(unittest.TestCase):
         self.assertItemsEqual(list(resp[CONT4][BLOCK]), ["blockSize", "locations"])
         self.assertIsNone(resp[self.badDID])
 
+    def testStringDateToEpoch(self):
+        """
+        Test the stringDateToEpoch function
+        """
+        dateString = 'Thu, 29 Apr 2021 13:15:42 UTC'
+        self.assertEqual(1619694942, stringDateToEpoch(dateString))
+        dateString = 'Thu, 1 Jan 1970 01:00:00 UTC'
+        self.assertEqual(0, stringDateToEpoch(dateString))
+
 
 if __name__ == '__main__':
     unittest.main()