Repository: dmwm/wmcore
Issue: JobCreator crashing while processing EventBased ACDC jobs - submit6

Apparently I missed another detail in my previous patch #9131, because there might be MCFakeFiles uploaded to the ACDCServer with a single element in the lumi list. The current patch would empty it, as can be seen in the log below
```
2019-04-11 14:36:29,403:140171761039104:INFO:EventBased:ACDC job created with {'attachments': {}, 'jobgroup': None, 'taskType': 'reprocessing', 'owner': 'pdmvserv', 'fwjr': None, 'id': None, 'jobType': 'Production', 'estimatedJobTime': 30520.721662463704, 'estimatedDiskUsage': 454400.0, 'state_time': 1555011389, 'custom': {}, 'state': 'new', 'location': None, 'workflow': 'vlimant_ACDC0_task_HIG-RunIIFall17wmLHEGS-02728__v1_T_190411_192704_3966', 'cache_dir': None, 'estimatedMemoryUsage': 2300.0, 'fwjr_path': None, 'input_files': [{'runs': set([]), 'minrun': 1L, 'last_event': 0, 'newlocations': set(['T3_US_PuertoRico', 'T2_FI_HIP', 'T2_UK_SGrid_RALPP', 'T3_US_Baylor', 'T3_TW_NTU_HEP', 'T2_KR_KNU', 'T2_RU_SINP', 'T3_US_UMD', 'T3_US_Colorado', 'T1_UK_RAL_Disk', 'T2_TH_CUNSTDA', 'T3_US_Kansas', 'T1_IT_CNAF_Disk', 'T2_IT_Bari', 'T2_US_UCSD', 'T2_RU_IHEP', 'T3_US_Vanderbilt_EC2', 'T3_US_JHU', 'T3_BY_NCPHEP', 'T3_US_TTU', 'T3_US_UCR', 'T3_TW_NCU', 'T2_CH_CSCS', 'T2_UA_KIPT', 'T2_PK_NCP', 'T2_RU_PNPI', 'T3_US_UCD', 'T3_CO_Uniandes', 'T2_FR_IPHC', 'T3_US_OSU', 'T3_GR_IASA_GR', 'T3_US_TAMU', 'T2_IT_Rome', 'T2_UK_London_Brunel', 'T3_IN_PUHEP', 'T3_IT_Trieste', 'T2_EE_Estonia', 'T2_IN_TIFR', 'T2_CN_Beijing', 'T2_US_Florida', 'T3_US_Princeton_ICSE', 'T3_US_Rutgers', 'T2_US_Wisconsin', 'T2_HU_Budapest', 'T2_DE_RWTH', 'T2_BR_SPRACE', 'T2_CH_CERN', 'T2_BR_UERJ', 'T3_MX_Cinvestav', 'T3_US_FNALLPC', 'T3_RU_FIAN', 'T3_US_Cornell', 'T2_ES_IFCA', 'T3_ES_Oviedo', 'T3_US_NotreDame', 'T2_DE_DESY', 'T3_HU_Debrecen', 'T2_US_Caltech', 'T3_FR_IPNL', 'T3_BG_UNI_SOFIA', 'T1_RU_JINR_Disk', 'T3_CN_PKU', 'T2_UK_London_IC', 'T2_US_Nebraska', 'T2_ES_CIEMAT', 'T1_FR_CCIN2P3_Disk', 'T2_KR_KISTI', 'T3_US_FSU', 'T3_KR_UOS', 'T3_IT_Perugia', 'T1_ES_PIC_Disk', 'T2_CH_CERNBOX', 'T2_TR_METU', 'T2_AT_Vienna', 'T2_US_Purdue', 'T3_US_Rice', 'T2_TW_NCHC', 'T2_BE_UCL', 'T3_US_FIT', 'T2_UK_SGrid_Bristol', 'T2_PT_NCG_Lisbon', 'T2_IT_Legnaro', 'T1_DE_KIT_Disk', 'T2_RU_ITEP', 'T3_US_SDSC', 'T3_US_FIU', 'T2_RU_JINR', 'T2_IT_Pisa', 'T2_GR_Ioannina', 'T3_US_MIT', 'T2_MY_UPM_BIRUNI', 'T2_FR_GRIF_IRFU', 'T3_US_UMiss', 'T2_FR_CCIN2P3', 'T2_PL_Warsaw', 'T2_US_MIT', 'T2_BE_IIHE', 'T2_RU_INR', 'T2_PL_Swierk', 'T3_CH_PSI', 'T3_IT_Bologna']), 'lfn': 'MCFakeFile-pdmvserv_task_HIG-RunIIFall17wmLHEGS-02728__v1_T_181024_163235_6575-HIG-RunIIFall17wmLHEGS-02728_0-e1016b623be5254da2c8ff2269c4e903', 'locations': set(['T3_US_PuertoRico', 'T2_FI_HIP', 'T2_UK_SGrid_RALPP', 'T3_US_Baylor', 'T3_TW_NTU_HEP', 'T2_KR_KNU', 'T2_RU_SINP', 'T3_US_UMD', 'T3_US_Colorado', 'T1_UK_RAL_Disk', 'T2_TH_CUNSTDA', 'T3_US_Kansas', 'T1_IT_CNAF_Disk', 'T2_IT_Bari', 'T2_US_UCSD', 'T2_RU_IHEP', 'T3_US_Vanderbilt_EC2', 'T3_US_JHU', 'T3_BY_NCPHEP', 'T3_US_TTU', 'T3_US_UCR', 'T3_TW_NCU', 'T2_CH_CSCS', 'T2_UA_KIPT', 'T2_PK_NCP', 'T2_RU_PNPI', 'T3_US_UCD', 'T3_CO_Uniandes', 'T2_FR_IPHC', 'T3_US_OSU', 'T3_GR_IASA_GR', 'T3_US_TAMU', 'T2_IT_Rome', 'T2_UK_London_Brunel', 'T3_IN_PUHEP', 'T3_IT_Trieste', 'T2_EE_Estonia', 'T2_IN_TIFR', 'T2_CN_Beijing', 'T2_US_Florida', 'T3_US_Princeton_ICSE', 'T3_US_Rutgers', 'T2_US_Wisconsin', 'T2_HU_Budapest', 'T2_DE_RWTH', 'T2_BR_SPRACE', 'T2_CH_CERN', 'T2_BR_UERJ', 'T3_MX_Cinvestav', 'T3_US_FNALLPC', 'T3_RU_FIAN', 'T3_US_Cornell', 'T2_ES_IFCA', 'T3_ES_Oviedo', 'T3_US_NotreDame', 'T2_DE_DESY', 'T3_HU_Debrecen', 'T2_US_Caltech', 'T3_FR_IPNL', 'T3_BG_UNI_SOFIA', 'T1_RU_JINR_Disk', 'T3_CN_PKU', 'T2_UK_London_IC', 'T2_US_Nebraska', 'T2_ES_CIEMAT', 'T1_FR_CCIN2P3_Disk', 'T2_KR_KISTI', 'T3_US_FSU', 'T3_KR_UOS', 'T3_IT_Perugia', 'T1_ES_PIC_Disk', 'T2_CH_CERNBOX', 'T2_TR_METU', 'T2_AT_Vienna', 'T2_US_Purdue', 'T3_US_Rice', 'T2_TW_NCHC', 'T2_BE_UCL', 'T3_US_FIT', 'T2_UK_SGrid_Bristol', 'T2_PT_NCG_Lisbon', 'T2_IT_Legnaro', 'T1_DE_KIT_Disk', 'T2_RU_ITEP', 'T3_US_SDSC', 'T3_US_FIU', 'T2_RU_JINR', 'T2_IT_Pisa', 'T2_GR_Ioannina', 'T3_US_MIT', 'T2_MY_UPM_BIRUNI', 'T2_FR_GRIF_IRFU', 'T3_US_UMiss', 'T2_FR_CCIN2P3', 'T2_PL_Warsaw', 'T2_US_MIT', 'T2_BE_IIHE', 'T2_RU_INR', 'T2_PL_Swierk', 'T3_CH_PSI', 'T3_IT_Bologna']), 'id': 12053720, 'first_event': 0, 'parents': set([]), 'checksums': {}, 'events': 6400, 'merged': True, 'size': 0}], 'task': '/vlimant_ACDC0_task_HIG-RunIIFall17wmLHEGS-02728__v1_T_190411_192704_3966/HIG-RunIIFall17wmLHEGS-02728_0', 'name': 'a8776c29-09e0-43de-9e6b-2edaa49c948f-1190', 'couch_record': None, 'mask': {'LastRun': 1, 'FirstRun': 1, 'inclusivemask': True, 'runAndLumis': {}, 'LastEvent': 8804801, 'FirstEvent': 8804001, 'LastLumi': 10895, 'FirstLumi': 10894}, 'retry_count': 0, 'sandbox': None, 'outcome': 'failure'}
Traceback (most recent call last):
  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMComponent/JobCreator/JobCreatorPoller.py", line 414, in algorithm
    self.pollSubscriptions()
  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMComponent/JobCreator/JobCreatorPoller.py", line 549, in pollSubscriptions
    wmbsJobGroups = next(jobSplittingFunction)
  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMComponent/JobCreator/JobCreatorPoller.py", line 87, in runSplitter
    groups = jobFactory(**splitParams)
  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMCore/JobSplitting/JobFactory.py", line 94, in __call__
    self.algorithm(*args, **kwargs)
  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMCore/JobSplitting/EventBased.py", line 115, in algorithm
    deterministicPileup, totalJobs)
  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMCore/JobSplitting/EventBased.py", line 235, in createACDCJobs
    currentLumi = acdcFile["lumis"][0]
IndexError: list index out of range
2019-04-11 14:36:29,480:140171761039104:ERROR:BaseWorkerThread:Error in worker algorithm (1):
Backtrace:
  <WMComponent.JobCreator.JobCreatorPoller.JobCreatorPoller object at 0x7f7c48341950> <@========== WMException Start ==========@>
Exception Class: JobCreatorException
Message: Failed to execute JobCreator. Error: list index out of range
        ModuleName : WMComponent.JobCreator.JobCreatorPoller
        MethodName : algorithm
        ClassInstance : None
        FileName : /data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMComponent/JobCreator/JobCreatorPoller.py
        ClassName : None
        LineNumber : 435
        ErrorNr : 0
Traceback: 
  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMComponent/JobCreator/JobCreatorPoller.py", line 414, in algorithm
    self.pollSubscriptions()

  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMComponent/JobCreator/JobCreatorPoller.py", line 549, in pollSubscriptions
    wmbsJobGroups = next(jobSplittingFunction)

  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMComponent/JobCreator/JobCreatorPoller.py", line 87, in runSplitter
    groups = jobFactory(**splitParams)

  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMCore/JobSplitting/JobFactory.py", line 94, in __call__
    self.algorithm(*args, **kwargs)

  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMCore/JobSplitting/EventBased.py", line 115, in algorithm
    deterministicPileup, totalJobs)

  File "/data/srv/wmagent/v1.1.20.patch3/sw/slc7_amd64_gcc630/cms/wmagent/1.1.20.patch3/lib/python2.7/site-packages/WMCore/JobSplitting/EventBased.py", line 235, in createACDCJobs
    currentLumi = acdcFile["lumis"][0]
```

Changed files:
src/python/WMCore/JobSplitting/EventBased.py, src/python/WMCore/ACDC/DataCollectionService.py

diff --git a/src/python/WMCore/ACDC/DataCollectionService.py b/src/python/WMCore/ACDC/DataCollectionService.py
index f83213312..026dff34e 100644
--- a/src/python/WMCore/ACDC/DataCollectionService.py
+++ b/src/python/WMCore/ACDC/DataCollectionService.py
@@ -46,7 +46,8 @@ def mergeFilesInfo(chunkFiles):
             # ErrorHandler expands that in the ACDC document as lumis : [9, 10] sigh...
             firstLumi = min(acdcFile['runs'][0]['lumis'])
             lastLumi = max(acdcFile['runs'][0]['lumis'])
-            acdcFile['runs'][0]['lumis'] = range(firstLumi, lastLumi)
+            if firstLumi != lastLumi:
+                acdcFile['runs'][0]['lumis'] = range(firstLumi, lastLumi)
             if fName not in mergedFiles:
                 mergedFiles[fName] = acdcFile
             else:
diff --git a/src/python/WMCore/JobSplitting/EventBased.py b/src/python/WMCore/JobSplitting/EventBased.py
index 48bb364df..5696b06a4 100644
--- a/src/python/WMCore/JobSplitting/EventBased.py
+++ b/src/python/WMCore/JobSplitting/EventBased.py
@@ -257,7 +257,7 @@ class EventBased(JobFactory):
                                                          disk=diskRequired)
                     if deterministicPileup:
                         self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob)
-                    logging.info("ACDC job created with %s", self.currentJob)
+                    logging.info("ACDC job created with mask: %s", self.currentJob['mask'])
                     eventsToRun -= eventsPerJob
                     currentEvent += eventsPerJob
                     totalJobs += 1