Repository: dmwm/wmcore
Issue: Python3 ReqMgr2 fails to set pickledarguments during assignment

**Impact of the bug**
ReqMgr2

**Describe the bug**
Note that this seem to be a rare case, since I managed to inject all the DMWM/Integration tests and a few old requests already. However, when I try to assign this workflow:
amaltaro_SC_ReDigi_Harvest_Agent136_Val_200814_105846_7752

it fails with a `400 Bad Request`. The logs extracted from ReqMgr2 testbed POD can be seen in [1]

**How to reproduce it**
Not sure, there must be something very peculiar about this one workflow(!)

**Expected behavior**
Python3 pickle requires data to be a bytes-like object, instead of python3 str. So we should encode the data we pass over to that pickle.loads call.

**Additional context and error message**
[1]
```
[30/Jun/2021:09:12:11]  Updating request "amaltaro_SC_ReDigi_Harvest_Agent136_Val_200814_105846_7752" with these user-provided args: {'SoftTimeout': 129600, 'UnmergedLFNBase': '/store/unmerged', 'TrustPUSitelists': True, 'GracePeriod': 30
0, 'SiteWhitelist': ['T1_US_FNAL', 'T2_CH_CERN'], 'MergedLFNBase': '/store/backfill/1', 'Team': 'testbed-vocms0193', 'ProcessingVersion': 21, 'TrustSitelists': False, 'ProcessingString': 'HG2107_Val_OLD_Alanv3', 'Dashboard': 'test', 'Site
Blacklist': [], 'Override': {'eos-lfn-prefix': 'root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED'}, 'AcquisitionEra': 'CMSSW_10_6_1', 'RequestStatus': 'assigned'}
Getting from Cache due to: CouchNotFoundError - reason: Object Not Found, data: {} result: None
[30/Jun/2021:09:12:11]  Assign request amaltaro_SC_ReDigi_Harvest_Agent136_Val_200814_105846_7752, input args: {'SoftTimeout': 129600, 'UnmergedLFNBase': '/store/unmerged', 'TrustPUSitelists': True, 'GracePeriod': 300, 'SiteWhitelist': ['
T1_US_FNAL', 'T2_CH_CERN'], 'MergedLFNBase': '/store/backfill/1', 'Team': 'testbed-vocms0193', 'ProcessingVersion': 21, 'TrustSitelists': False, 'ProcessingString': 'HG2107_Val_OLD_Alanv3', 'Dashboard': 'test', 'SiteBlacklist': [], 'Overr
ide': {'eos-lfn-prefix': 'root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED'}, 'AcquisitionEra': 'CMSSW_10_6_1', 'RequestStatus': 'assigned', 'HardTimeout': 129900} ...
[30/Jun/2021:09:12:11]  Error for request args {'SoftTimeout': 129600, 'UnmergedLFNBase': '/store/unmerged', 'TrustPUSitelists': True, 'GracePeriod': 300, 'SiteWhitelist': ['T1_US_FNAL', 'T2_CH_CERN'], 'MergedLFNBase': '/store/backfill/1'
, 'Team': 'testbed-vocms0193', 'ProcessingVersion': 21, 'TrustSitelists': False, 'ProcessingString': 'HG2107_Val_OLD_Alanv3', 'Dashboard': 'test', 'SiteBlacklist': [], 'Override': {'eos-lfn-prefix': 'root://eoscms.cern.ch//eos/cms/store/l
ogs/prod/recent/TESTBED'}, 'AcquisitionEra': 'CMSSW_10_6_1', 'RequestStatus': 'assigned', 'HardTimeout': 129900, 'EventStreams': 0, 'CustodialSites': [], 'NonCustodialSites': [], 'AutoApproveSubscriptionSites': [], 'CustodialSubType': 'Re
plica', 'NonCustodialSubType': 'Replica', 'CustodialGroup': 'DataOps', 'NonCustodialGroup': 'DataOps', 'SubscriptionPriority': 'Low', 'DeleteFromSource': False, 'MinMergeSize': 2147483648, 'MaxMergeSize': 4294967296, 'MaxWaitTime': 86400,
 'MaxMergeEvents': 100000000, 'AllowOpportunistic': False, 'BlockCloseMaxWaitTime': 66400, 'BlockCloseMaxFiles': 500, 'BlockCloseMaxEvents': 25000000, 'BlockCloseMaxSize': 5000000000000, 'ChainParentageMap': {}}: Traceback (most recent ca
ll last):
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/ReqMgr/Service/Request.py", line 436, in _handleAssignmentStateTransition
    workload.updateArguments(request_args)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMWorkload.py", line 1969, in updateArguments
    self.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMWorkload.py", line 1154, in setLFNBase
    self.updateLFNsAndDatasets(runNumber=runNumber)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMWorkload.py", line 806, in updateLFNsAndDatasets
    task.updateLFNsAndDatasets(runNumber=runNumber)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMTask.py", line 1783, in updateLFNsAndDatasets
    task.updateLFNsAndDatasets(runNumber=runNumber)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMTask.py", line 1774, in updateLFNsAndDatasets
    self.updateDatasetName(datasetName)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/WMTask.py", line 1801, in updateDatasetName
    cmsswHelper.setDatasetName(datasetName)
  File "/data/srv/HG2107b/sw/slc7_amd64_gcc630/cms/reqmgr2/1.5.0.pre5/lib/python3.8/site-packages/WMCore/WMSpec/Steps/Templates/CMSSW.py", line 241, in setDatasetName
    args = pickle.loads(self.data.application.configuration.pickledarguments)
TypeError: a bytes-like object is required, not 'str'
```

Changed files:
src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py, src/python/PSetTweaks/WMTweak.py

diff --git a/src/python/PSetTweaks/WMTweak.py b/src/python/PSetTweaks/WMTweak.py
index 6ab83bad0..e1b998eb2 100644
--- a/src/python/PSetTweaks/WMTweak.py
+++ b/src/python/PSetTweaks/WMTweak.py
@@ -15,7 +15,8 @@ from future.utils import viewitems, viewkeys
 import logging
 import os
 import pickle
-
+from Utils.PythonVersion import PY3
+from Utils.Utilities import encodeUnicodeToBytesConditional
 from PSetTweaks.PSetTweak import PSetTweak
 
 # params to be extracted from an output module
@@ -388,7 +389,9 @@ def makeTaskTweak(stepSection, result):
     if hasattr(stepSection, "application"):
         if hasattr(stepSection.application, "configuration"):
             if hasattr(stepSection.application.configuration, "pickledarguments"):
-                args = pickle.loads(stepSection.application.configuration.pickledarguments)
+                pklArgs = encodeUnicodeToBytesConditional(stepSection.application.configuration.pickledarguments,
+                                                          condition=PY3)
+                args = pickle.loads(pklArgs)
                 if 'globalTag' in args:
                     result.addParameter("process.GlobalTag.globaltag",  "customTypeCms.string('%s')" % args['globalTag'])
                 if 'globalTagTransaction' in args:
diff --git a/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py b/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py
index 758243a58..e339ceeac 100644
--- a/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py
+++ b/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py
@@ -15,7 +15,8 @@ import socket
 
 from PSetTweaks.PSetTweak import PSetTweak
 from PSetTweaks.WMTweak import  makeJobTweak, makeOutputTweak, makeTaskTweak, resizeResources
-from Utils.Utilities import decodeBytesToUnicode
+from Utils.PythonVersion import PY3
+from Utils.Utilities import decodeBytesToUnicode, encodeUnicodeToBytesConditional
 from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig
 from WMCore.Storage.TrivialFileCatalog import TrivialFileCatalog
 from WMCore.WMRuntime.ScriptInterface import ScriptInterface
@@ -487,7 +488,9 @@ class SetupCMSSWPset(ScriptInterface):
             os.path.join(self.stepSpace.location, self.configPickle))
 
         if hasattr(self.step.data.application.configuration, "pickledarguments"):
-            args = pickle.loads(self.step.data.application.configuration.pickledarguments)
+            pklArgs = encodeUnicodeToBytesConditional(self.step.data.application.configuration.pickledarguments,
+                                                      condition=PY3)
+            args = pickle.loads(pklArgs)
             datasetName = args.get('datasetName', None)
         if datasetName:
             cmd += " --datasetName %s" % (datasetName)
@@ -681,7 +684,9 @@ class SetupCMSSWPset(ScriptInterface):
         if scenario is not None and scenario != "":
             self.logger.info("Setting up job scenario/process")
             if getattr(self.step.data.application.configuration, "pickledarguments", None) is not None:
-                funcArgs = pickle.loads(self.step.data.application.configuration.pickledarguments)
+                pklArgs = encodeUnicodeToBytesConditional(self.step.data.application.configuration.pickledarguments,
+                                                          condition=PY3)
+                funcArgs = pickle.loads(pklArgs)
             else:
                 funcArgs = {}