Repository: dmwm/wmcore
Issue: MS: do we consider block white/black list in MSTransferor

**Impact of the bug**
ReqMgr2MS

**Describe the bug**
I am actually not sure it's a bug or not. It might be that code is already in place and this issue is not valid; anyhow, just so I don't forget.
Do we consider BlockWhitelist and BlockBlacklist for input data placement of workflows without any lumi and run lists? This code basically:
https://github.com/dmwm/WMCore/blob/master/src/python/WMCore/MicroService/Unified/RequestInfo.py#L355-L363

**How to reproduce it**
Steps to reproduce the behavior:

**Expected behavior**
A clear and concise description of what you expected to happen.

**Additional context and error message**
Add any other context about the problem here, like error message and/or traceback. You might want to use triple back ticks to properly format it.


Changed files:
src/python/WMCore/MicroService/Unified/RequestInfo.py, src/python/WMCore/MicroService/DataStructs/Workflow.py

diff --git a/src/python/WMCore/MicroService/DataStructs/Workflow.py b/src/python/WMCore/MicroService/DataStructs/Workflow.py
index 4a23ecab3..a95ea9307 100644
--- a/src/python/WMCore/MicroService/DataStructs/Workflow.py
+++ b/src/python/WMCore/MicroService/DataStructs/Workflow.py
@@ -5,6 +5,7 @@ required by MS Transferor
 from __future__ import division, print_function
 
 import operator
+from copy import copy, deepcopy
 from WMCore.DataStructs.LumiList import LumiList
 from WMCore.MicroService.Unified.Common import getMSLogger, gigaBytes
 
@@ -213,7 +214,16 @@ class Workflow(object):
         {"block_name": {"blockSize": 1234,
                         "locations": [list of locations]}}
         """
-        self.primaryBlocks = blocksDict
+        blockWhite = self.getBlockWhitelist()
+        if blockWhite:
+            for block in blockWhite:
+                if block in blocksDict:
+                    self.primaryBlocks[block] = copy(blocksDict[block])
+        else:
+            self.primaryBlocks = deepcopy(blocksDict)
+
+        for block in self.getBlockBlacklist():
+            self.primaryBlocks.pop(block, None)
 
     def getPrimaryBlocks(self):
         """
@@ -263,16 +273,14 @@ class Workflow(object):
         Sets a relationship between input primary block and its parent block(s)
         This method guarantees that only parent blocks with valid replicas are
         kept around (and later transferred)
+        The child block list is also final, meaning all the run/block/lumi lists
+        have already been applied.
         :param blocksDict: dict key'ed by the primary block, with a list of parent blocks
         """
         # flat list with the final parent blocks
         parentBlocks = set()
         # remove parent blocks without any valid replica (only invalid files)
         for child, parents in blocksDict.items():
-            if child not in self.getPrimaryBlocks():
-                # then we don't need this child+parent data
-                continue
-
             for parent in list(parents):
                 if parent not in self.getParentBlocks():
                     # then drop this block
diff --git a/src/python/WMCore/MicroService/Unified/RequestInfo.py b/src/python/WMCore/MicroService/Unified/RequestInfo.py
index 93f1984e5..17b30bf77 100644
--- a/src/python/WMCore/MicroService/Unified/RequestInfo.py
+++ b/src/python/WMCore/MicroService/Unified/RequestInfo.py
@@ -306,14 +306,14 @@ class RequestInfo(MSCore):
             for dataIn in wflow.getDataCampaignMap():
                 if dataIn['type'] == "primary":
                     newBlockDict = self._handleInputDataInfo(wflow, dataIn['name'],
-                                                             blocksByDset[dataIn['name']])
+                                                             blocksByDset[dataIn['name']], True)
                     wflow.setPrimaryBlocks(newBlockDict)
                 elif dataIn['type'] == "parent":
                     newBlockDict = self._handleInputDataInfo(wflow, dataIn['name'],
-                                                             blocksByDset[dataIn['name']])
+                                                             blocksByDset[dataIn['name']], False)
                     wflow.setParentBlocks(newBlockDict)
 
-    def _handleInputDataInfo(self, wflow, dset, blocksDict):
+    def _handleInputDataInfo(self, wflow, dset, blocksDict, primary):
         """
         Handle primary input data, such that we can also consider a
         Run white/black lists, and also compare against
@@ -328,7 +328,6 @@ class RequestInfo(MSCore):
         :return: dictionary of block names and block size
         """
         # FIXME: this is heavy and should be done concurrently
-        finalBlocks = {}
         dbsUrl = wflow.getDbsUrl()
         runList = wflow.getRunlist()
         lumiList = wflow.getLumilist()
@@ -339,9 +338,10 @@ class RequestInfo(MSCore):
                 runList.append(int(run))
             runList = list(set(runList))
         if runList:
+            finalBlocks = copy(blocksDict)
             # Run number 1 is not supported by DBSServer
             if int(runList[0]) == 1:
-                finalBlocks = copy(blocksDict)
+                pass
             else:
                 self.logger.info("Fetching blocks matching a list of runs for %s", wflow.getName())
                 # then find blocks matching that run list
@@ -352,15 +352,18 @@ class RequestInfo(MSCore):
                         finalBlocks[block] = blocksDict[block]
                     else:
                         self.logger.info("Dropping block with no replicas in PhEDEx: %s", block)
-            # now apply filters based on the blocks white and black list
-            whiteBlocks = wflow.getBlockWhitelist()
-            if whiteBlocks:
-                for block in list(finalBlocks):
-                    if block not in whiteBlocks:
-                        finalBlocks.pop(block)
-            blackBlocks = wflow.getBlockBlacklist()
-            for block in blackBlocks:
-                finalBlocks.pop(block, None)
+
+            if primary:
+                # then apply block white/black list filters to avoid unnecessary calls
+                whiteBlocks = wflow.getBlockWhitelist()
+                if whiteBlocks:
+                    for block in list(finalBlocks):
+                        if block not in whiteBlocks:
+                            finalBlocks.pop(block)
+                blackBlocks = wflow.getBlockBlacklist()
+                for block in blackBlocks:
+                    finalBlocks.pop(block, None)
+
             if lumiList:
                 self.logger.info("Fetching block/lumi information for %d blocks in %s",
                                  len(finalBlocks), wflow.getName())