Repository: dmwm/wmcore
Issue: Python3 ReqMgr2 fails to set pickledarguments during assignment

**Impact of the bug**
ReqMgr2

**Describe the bug**
Note that this seem to be a rare case, since I managed to inject all the DMWM/Integration tests and a few old requests already. However, when I try to assign this workflow:
amaltaro_SC_ReDigi_Harvest_Agent136_Val_200814_105846_7752

it fails with a `400 Bad Request`. The logs extracted from ReqMgr2 testbed POD can be seen in [1]

**How to reproduce it**
Not sure, there must be something very peculiar about this one workflow(!)

**Expected behavior**
Python3 pickle requires data to be a bytes-like object, instead of python3 str. So we should encode the data we pass over to that pickle.loads call.

**Additional context and error message**
[1]
```
[30/Jun/2021:09:12:11]  Updating request "amaltaro_SC_ReDigi_Harvest_Agent136_Val_200814_105846_7752" with these user-provided args: {'SoftTimeout': 129600, 'UnmergedLFNBase': '/store/unmerged', 'TrustPUSitelists': True, 'GracePeriod': 30
0, 'SiteWhitelist': ['T1_US_FNAL', 'T2_CH_CERN'], 'MergedLFNBase': '/store/backfill/1', 'Team': 'testbed-vocms0193', 'ProcessingVersion': 21, 'TrustSitelists': False, 'ProcessingString': 'HG2107_Val_OLD_Alanv3', 'Dashboard': 'test', 'Site
Blacklist': [], 'Override': {'eos-lfn-prefix': 'root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED'}, 'AcquisitionEra': 'CMSSW_10_6_1', 'RequestStatus': 'assigned'}
Getting from Cache due to: CouchNotFoundError - reason: Object Not Found, data: {} result: None
[30/Jun/2021:09:12:11]  Assign request amaltaro_SC_ReDigi_Harvest_Agent136_Val_200814_105846_7752, input args: {'SoftTimeout': 129600, 'UnmergedLFNBase': '/store/unmerged', 'TrustPUSitelists': True, 'GracePeriod': 300, 'SiteWhitelist': ['
T1_US_FNAL', 'T2_CH_CERN'], 'MergedLFNBase': '/store/backfill/1', 'Team': 'testbed-vocms0193', 'ProcessingVersion': 21, 'TrustSitelists': False, 'ProcessingString': 'HG2107_Val_OLD_Alanv3', 'Dashboard': 'test', 'SiteBlacklist': [], 'Overr
ide': {'eos-lfn-prefix': 'root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED'}, 'AcquisitionEra': 'CMSSW_10_6_1', 'RequestStatus': 'assigned', 'HardTimeout': 129900} ...
[30/Jun/2021:09:12:11]  Error for request args {'SoftTimeout': 129600, 'UnmergedLFNBase': '/store/unmerged', 'TrustPUSitelists': True, 'GracePeriod': 300, 'SiteWhitelist': ['T1_US_FNAL', 'T2_CH_CERN'], 'MergedLFNBase': '/store/backfill/1'
, 'Team': 'testbed-vocms0193', 'ProcessingVersion': 21, 'TrustSitelists': False, 'ProcessingString': 'HG2107_Val_OLD_Alanv3', 'Dashboard': 'test', 'SiteBlacklist': [], 'Override': {'eos-lfn-prefix': 'root://eoscms.cern.ch//eos/cms/store/l
ogs/prod/recent/TESTBED'}, 'AcquisitionEra': 'CMSSW_10_6_1', 'RequestStatus': 'assigned', 'HardTimeout': 129900, 'EventStreams': 0, 'CustodialSites': [], 'NonCustodialSites': [], 'AutoApproveSubscriptionSites': [], 'CustodialSubType': 'Re
plica', 'NonCustodialSubType': 'Replica', 'CustodialGroup': 'DataOps', 'NonCustodialGroup': 'DataOps', 'SubscriptionPriority': 'Low', 'DeleteFromSource': False, 'MinMergeSize': 2147483648, 'MaxMergeSize': 4294967296, 'MaxWaitTime': 86400,
 'MaxMergeEvents': 100000000, 'AllowOpportunistic': False, 'BlockCloseMaxWaitTime': 66400, 'BlockCloseMaxFiles': 500, 'BlockCloseMaxEvents': 25000000, 'BlockCloseMaxSize': 5000000000000, 'ChainParentageMap': {}}: Traceback (most recent ca
ll last):
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/ReqMgr/Service/Request.py", line 436, in _handleAssignmentStateTransition
    workload.updateArguments(request_args)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMWorkload.py", line 1969, in updateArguments
    self.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMWorkload.py", line 1154, in setLFNBase
    self.updateLFNsAndDatasets(runNumber=runNumber)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMWorkload.py", line 806, in updateLFNsAndDatasets
    task.updateLFNsAndDatasets(runNumber=runNumber)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMTask.py", line 1783, in updateLFNsAndDatasets
    task.updateLFNsAndDatasets(runNumber=runNumber)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMTask.py", line 1774, in updateLFNsAndDatasets
    self.updateDatasetName(datasetName)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMTask.py", line 1801, in updateDatasetName
    cmsswHelper.setDatasetName(datasetName)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/Steps/Templates/CMSSW.py", line 241, in setDatasetName
    args = pickle.loads(self.data.application.configuration.pickledarguments)
TypeError: a bytes-like object is required, not 'str'
```

Changed files:
src/python/WMCore/WMSpec/Steps/Templates/CMSSW.py

diff --git a/src/python/WMCore/WMSpec/Steps/Templates/CMSSW.py b/src/python/WMCore/WMSpec/Steps/Templates/CMSSW.py
index 3daac77f3..7bb200c82 100644
--- a/src/python/WMCore/WMSpec/Steps/Templates/CMSSW.py
+++ b/src/python/WMCore/WMSpec/Steps/Templates/CMSSW.py
@@ -10,6 +10,7 @@ import pickle
 
 from future.utils import viewitems
 
+from Utils.Utilities import encodeUnicodeToBytes
 from WMCore.WMSpec.ConfigSectionTree import nodeName
 from WMCore.WMSpec.Steps.Template import CoreHelper, Template
 
@@ -208,7 +209,7 @@ class CMSSWStepHelper(CoreHelper):
 
         args = {}
         if hasattr(self.data.application.configuration, "pickledarguments"):
-            args = pickle.loads(self.data.application.configuration.pickledarguments)
+            args = pickle.loads(encodeUnicodeToBytes(self.data.application.configuration.pickledarguments))
         args['globalTag'] = globalTag
         # FIXME: once both central services and WMAgent are in Py3, we can remove protocol=0
         self.data.application.configuration.pickledarguments = pickle.dumps(args, protocol=0)
@@ -225,7 +226,8 @@ class CMSSWStepHelper(CoreHelper):
             if hasattr(self.data.application.configuration.arguments, "globalTag"):
                 return self.data.application.configuration.arguments.globalTag
 
-        return pickle.loads(self.data.application.configuration.pickledarguments)['globalTag']
+        pickledArgs = encodeUnicodeToBytes(self.data.application.configuration.pickledarguments)
+        return pickle.loads(pickledArgs)['globalTag']
 
     def setDatasetName(self, datasetName):
         """
@@ -238,7 +240,7 @@ class CMSSWStepHelper(CoreHelper):
 
         args = {}
         if hasattr(self.data.application.configuration, "pickledarguments"):
-            args = pickle.loads(self.data.application.configuration.pickledarguments)
+            args = pickle.loads(encodeUnicodeToBytes(self.data.application.configuration.pickledarguments))
         args['datasetName'] = datasetName
         # FIXME: once both central services and WMAgent are in Py3, we can remove protocol=0
         self.data.application.configuration.pickledarguments = pickle.dumps(args, protocol=0)