Repository: dmwm/wmcore
Issue: LumiList not working properly for TaskChain

If one provides the LumiList in the main dict of a TaskChain request, it will be taken into consideration for the job splitting and one can see the "splitting.lumis" parameter in the workload.
However, this is not how TaskChain works, these kind of parameters should always go inside a specific Task(s), which does not work in the current code.

One can use this request
https://github.com/dmwm/WMCore/blob/master/test/data/ReqMgr/requests/DMWM/TaskChain_Data.json
and add this line to the Task1 dict

```
       "LumiList": {"160960": [[277, 287], [79,85], [167,177], [226,232]]},
```

in order to reproduce this failure.


Changed files:
src/python/WMCore/WMSpec/WMTask.py, test/python/WMCore_t/WMSpec_t/StdSpecs_t/TaskChain_t.py, src/python/WMCore/WMSpec/StdSpecs/TaskChain.py, src/python/WMCore/WMSpec/WMWorkload.py

diff --git a/src/python/WMCore/WMSpec/StdSpecs/TaskChain.py b/src/python/WMCore/WMSpec/StdSpecs/TaskChain.py
index 8a9457da5..054fcbb4a 100644
--- a/src/python/WMCore/WMSpec/StdSpecs/TaskChain.py
+++ b/src/python/WMCore/WMSpec/StdSpecs/TaskChain.py
@@ -10,7 +10,7 @@ Provide support for building arbitrary chains of WMTasks based on a nested dicti
 starting with either a generation (create new MC events) or processing (use an existing input dataset) step, followed
 by a chain of dependent WMTasks that process the subsequent output.
 
-The request is formed as a dictionary where some global parameters are provided as normal, but the 
+The request is formed as a dictionary where some global parameters are provided as normal, but the
 processing tasks are specified as sub dictionaries.
 
 The top level dict should contain the parameter TaskChain and the value is the number of processing tasks to be run.
@@ -25,7 +25,7 @@ CouchDB parameters the main request parameters are:
     "Requestor": "sfoulkes@fnal.gov",                 Person responsible
     "GlobalTag": "GR10_P_v4::All",                    Global Tag
     "CouchURL": "http://couchserver.cern.ch",         URL of CouchDB containing ConfigCache (Used for all sub-tasks)
-    "ConfigCacheUrl": https://cmsweb-testbed.cern.ch/couchdb URL of an alternative CouchDB server containing Config documents     
+    "ConfigCacheUrl": https://cmsweb-testbed.cern.ch/couchdb URL of an alternative CouchDB server containing Config documents
     "CouchDBName": "config_cache",                    Name of Couch Database containing config cache (Used for all sub-tasks)
     "TaskChain" : 4,                                  Define number of tasks in chain.
 }
@@ -92,7 +92,7 @@ from WMCore.WMSpec.WMWorkloadTools import makeList, strToBool, validateArguments
 
 #
 # simple utils for data mining the request dictionary
-# 
+#
 isGenerator = lambda args: not args["Task1"].get("InputDataset", None)
 parentTaskModule = lambda args: args.get("InputFromOutputModule", None)
 
@@ -239,14 +239,14 @@ class TaskChainWorkloadFactory(StdBase):
             parentTask = None
             if parent in self.mergeMapping:
                 parentTask = self.mergeMapping[parent][parentTaskModule(taskConf)]
-                
+
             task = self.makeTask(taskConf, parentTask)
             if i == 1:
                 # First task will either be generator or processing
                 self.workload.setDashboardActivity("relval")
                 if isGenerator(arguments):
                     # generate mc events
-                    self.workload.setWorkQueueSplitPolicy("MonteCarlo", taskConf['SplittingAlgo'], 
+                    self.workload.setWorkQueueSplitPolicy("MonteCarlo", taskConf['SplittingAlgo'],
                                                           taskConf['SplittingArguments'])
                     self.workload.setEndPolicy("SingleShot")
                     self.setupGeneratorTask(task, taskConf)
@@ -262,21 +262,21 @@ class TaskChainWorkloadFactory(StdBase):
             self.taskMapping[task.name()] = taskConf
 
         self.workload.ignoreOutputModules(self.ignoredOutputModules)
-        
+
         # setting the parameters which need to be set for all the tasks
         # sets acquisitionEra, processingVersion, processingString
         self.workload.setTaskPropertiesFromWorkload()
-        return self.workload  
+        return self.workload
+
 
-            
     def makeTask(self, taskConf, parentTask = None):
         """
         _makeTask_
-        
-        create new Task and populate it with basic required parameters from the 
-        taskConfig provided, if parentTask is None, the task will be created in 
+
+        create new Task and populate it with basic required parameters from the
+        taskConfig provided, if parentTask is None, the task will be created in
         the workload, else the task will be created as a child of the parent Task
-        
+
         """
         if parentTask == None:
             task = self.workload.newTask(taskConf['TaskName'])
@@ -288,7 +288,7 @@ class TaskChainWorkloadFactory(StdBase):
     def setupGeneratorTask(self, task, taskConf):
         """
         _setupGeneratorTask_
-        
+
         Set up an initial generation task
         """
         cmsswStepType = "CMSSW"
@@ -335,6 +335,7 @@ class TaskChainWorkloadFactory(StdBase):
         splitArguments = taskConf["SplittingArguments"]
         keepOutput     = taskConf["KeepOutput"]
         transientModules = taskConf["TransientOutputModules"]
+        lumiMask = taskConf.get('LumiList', None)
         forceUnmerged = (not keepOutput) or (len(transientModules) > 0)
 
         # in case the initial task is a processing task, we have an input dataset, otherwise
@@ -390,6 +391,8 @@ class TaskChainWorkloadFactory(StdBase):
 
         if taskConf["PileupConfig"]:
             self.setupPileup(task, taskConf['PileupConfig'])
+        if lumiMask:
+            task.setLumiMask(lumiMask)
 
         self.addLogCollectTask(task, 'LogCollectFor%s' % task.name())
         self.setUpMergeTasks(task, outputMods, splitAlgorithm,
@@ -585,7 +588,7 @@ class TaskChainWorkloadFactory(StdBase):
     def validateSchema(self, schema):
         """
         _validateSchema_
-        
+
         Go over each task and make sure it matches validation
         parameters derived from Dave's requirements.
         """
diff --git a/src/python/WMCore/WMSpec/WMTask.py b/src/python/WMCore/WMSpec/WMTask.py
index c1b889de5..52fb8f5f5 100644
--- a/src/python/WMCore/WMSpec/WMTask.py
+++ b/src/python/WMCore/WMSpec/WMTask.py
@@ -1215,7 +1215,7 @@ class WMTaskHelper(TreeHelper):
         """
         return getattr(self.data.parameters, 'acquisitionEra', None)
 
-    def setLumiMask(self, lumiMask = {}):
+    def setLumiMask(self, lumiMask = {}, override = True):
         """
         Attach the given LumiMask to the task
         At this point the lumi mask is just the compactList dict not the LumiList object
@@ -1224,6 +1224,11 @@ class WMTaskHelper(TreeHelper):
         if not lumiMask:
             return
 
+        runs = getattr(self.data.input.splitting, 'runs', None)
+        lumis = getattr(self.data.input.splitting, 'lumis', None)
+        if not override and runs and lumis: # Unless instructed, don't overwrite runs and lumis which may be there from a task already
+            return
+
         runs = []
         lumis = []
         for run, runLumis in lumiMask.items():
diff --git a/src/python/WMCore/WMSpec/WMWorkload.py b/src/python/WMCore/WMSpec/WMWorkload.py
index 034815585..b1edf7b27 100644
--- a/src/python/WMCore/WMSpec/WMWorkload.py
+++ b/src/python/WMCore/WMSpec/WMWorkload.py
@@ -807,7 +807,7 @@ class WMWorkloadHelper(PersistencyHelper):
             taskIterator = self.taskIterator()
 
         for task in taskIterator:
-            task.setLumiMask(lumiLists)
+            task.setLumiMask(lumiLists, override=False)
             self.setLumiList(lumiLists, task)
 
         #set lumiList for workload (need to refactor)
diff --git a/test/python/WMCore_t/WMSpec_t/StdSpecs_t/TaskChain_t.py b/test/python/WMCore_t/WMSpec_t/StdSpecs_t/TaskChain_t.py
index ebec884ce..c430b245b 100644
--- a/test/python/WMCore_t/WMSpec_t/StdSpecs_t/TaskChain_t.py
+++ b/test/python/WMCore_t/WMSpec_t/StdSpecs_t/TaskChain_t.py
@@ -473,6 +473,7 @@ class TaskChainTests(unittest.TestCase):
         processorDocs = makeProcessingConfigs(self.configDatabase)
         testArguments = TaskChainWorkloadFactory.getTestArguments()
         lumiDict = {"1":[[2,4], [8,50]], "2":[[100,200], [210,210]]}
+        lumiDict2 = {"1":[[2,4], [8,40]], "2":[[100,150], [210,210]]}
         arguments = {
             "AcquisitionEra": "ReleaseValidation",
             "Requestor": "sfoulkes@fnal.gov",
@@ -503,6 +504,7 @@ class TaskChainTests(unittest.TestCase):
                 "CMSSWVersion" : "CMSSW_3_1_2",
                 "ScramArch" : "CompatibleRECOArch",
                 "PrimaryDataset" : "ZeroBias",
+                "LumiList": lumiDict2,
             },
             "Task3" : {
                 "TaskName" : "ALCAReco",
@@ -553,8 +555,9 @@ class TaskChainTests(unittest.TestCase):
         self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
         self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])
 
+        # Make sure this task has a different lumilist than the global one
         reco = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
-        self.assertEqual(lumiDict, reco.getLumiMask())
+        self.assertEqual(lumiDict2, reco.getLumiMask())
         recoStep = reco.getStepHelper("cmsRun1")
         self.assertEqual(recoStep.getGlobalTag(), arguments['Task2']['GlobalTag'])
         self.assertEqual(recoStep.getCMSSWVersion(), arguments['Task2']['CMSSWVersion'])