Repository: dmwm/wmcore
Issue: Get only valid files from dbs

Updates agent only after https://github.com/dmwm/DBS/issues/445 is in production


Changed files:
src/python/WMCore/WorkQueue/Policy/Start/Block.py, src/python/WMQuality/Emulators/DBSClient/DBS3Reader.py, src/python/WMCore/Services/DBS/DBS3Reader.py

diff --git a/src/python/WMCore/Services/DBS/DBS3Reader.py b/src/python/WMCore/Services/DBS/DBS3Reader.py
index 1faf4e822..ef2919a91 100644
--- a/src/python/WMCore/Services/DBS/DBS3Reader.py
+++ b/src/python/WMCore/Services/DBS/DBS3Reader.py
@@ -189,33 +189,8 @@ class DBS3Reader:
 
         """
         return [ x['logical_file_name'] for x in self.dbs.listFiles(dataset = datasetPath)]
-
-    def _listValidDatasetFiles(self, datasetPath, onlyLFN = False):
-        """
-        _listValidDatasetFiles_
-
-        Get a list of only valid files for dataset
-        """
-        allFiles = self.dbs.listFiles(dataset = datasetPath, detail=True)
-        
-        if onlyLFN:
-            return [x['logical_file_name'] for x in allFiles if x['is_file_valid']]
-        else:
-            return [x for x in allFiles if x['is_file_valid']]
         
 
-    def _listValidBlockFiles(self, fileBlockName, onlyLFN = False):
-        """
-        _listValidBlockFiles_
-
-        Get a list of only valid files for dataset
-        """
-        allFiles = self.dbs.listFiles(block_name = fileBlockName, detail = True)
-        if onlyLFN:
-            return [x['logical_file_name'] for x in allFiles if x['is_file_valid']]
-        else:
-            return [x for x in allFiles if x['is_file_valid']]
-
     def listDatasetFileDetails(self, datasetPath, getParents=False):
         """
         _listDatasetFileDetails_
@@ -232,7 +207,7 @@ class DBS3Reader:
             }
 
         """
-        fileDetails = self._listValidDatasetFiles(datasetPath)
+        fileDetails = self.dbs.listFiles(dataset = datasetPath, validFileOnly = 1, detail=True)
         blocks = set() #the set of blocks of the dataset
         #Iterate over the files and prepare the set of blocks and a dict where the keys are the files
         files = {}
@@ -256,7 +231,7 @@ class DBS3Reader:
                 for p in parents:
                     files[p['logical_file_name']]['Parents'].extend(p['parent_logical_file_name'])
             #get the lumis
-            file_lumis = self.dbs.listFileLumis(block_name=blockName)
+            file_lumis = self.dbs.listFileLumis(block_name=blockName, validFileOnly = 1)
             for f in file_lumis:
                 if f['run_num'] in files[f['logical_file_name']]['Lumis']:
                     files[f['logical_file_name']]['Lumis'][f['run_num']].extend(f['lumi_section_num'])
@@ -276,7 +251,7 @@ class DBS3Reader:
         Return the list of lfns that are in the dataset
 
         """
-        allLfns = self._listValidDatasetFiles(datasetPath, onlyLFN = True)
+        allLfns = self.dbs.listFiles(dataset = datasetPath, validFileOnly = 1, detail = False)
         setOfAllLfns = set(allLfns)
         setOfKnownLfns = set(lfns)
         return list(setOfAllLfns.intersection(setOfKnownLfns))
@@ -289,7 +264,7 @@ class DBS3Reader:
         are *not* known by DBS
 
         """
-        allLfns = self._listValidDatasetFiles(datasetPath, onlyLFN = True)
+        allLfns = self.dbs.listFiles(dataset = datasetPath, validFileOnly = 1, detail = False)
         setOfAllLfns = set(allLfns)
         setOfKnownLfns = set(lfns)
         knownFiles = setOfAllLfns.intersection(setOfKnownLfns)
@@ -306,9 +281,9 @@ class DBS3Reader:
             self.checkDatasetPath(dataset)
         try:
             if block:
-                summary = self.dbs.listFileSummaries(block_name = block)
+                summary = self.dbs.listFileSummaries(block_name = block, validFileOnly = 1)
             else: # dataset case dataset shouldn't be None
-                summary = self.dbs.listFileSummaries(dataset = dataset)
+                summary = self.dbs.listFileSummaries(dataset = dataset, validFileOnly = 1)
         except dbsClientException, ex:
             msg = "Error in DBSReader.listDatasetSummary(%s, %s)\\n" % (dataset, block)
             msg += "%s\\n" % formatEx3(ex)
@@ -447,7 +422,7 @@ class DBS3Reader:
             raise DBSReaderError(msg % fileBlockName)
 
         try:
-            files = self._listValidBlockFiles(fileBlockName)
+            files = self.dbs.listFiles(block_name = fileBlockName, validFileOnly = 1, detail = True)
         except dbsClientException, ex:
             msg = "Error in "
             msg += "DBSReader.listFilesInBlock(%s)\\n" % fileBlockName
@@ -456,7 +431,7 @@ class DBS3Reader:
 
         if lumis:
             try:
-                lumiLists = self.dbs.listFileLumis(block_name = fileBlockName)
+                lumiLists = self.dbs.listFileLumis(block_name = fileBlockName, validFileOnly = 1)
             except dbsClientException, ex:
                 msg = "Error in "
                 msg += "DBSReader.listFileLumis(%s)\\n" % fileBlockName
@@ -522,7 +497,7 @@ class DBS3Reader:
             raise DBSReaderError(msg % fileBlockName)
 
         try:
-            lfns = self._listValidBlockFiles(fileBlockName, onlyLFN = True)
+            lfns =  self.dbs.listFiles(block_name = fileBlockName, validFileOnly = 1, detail = False)
             return lfns
         except dbsClientException, ex:
             msg = "Error in "
diff --git a/src/python/WMCore/WorkQueue/Policy/Start/Block.py b/src/python/WMCore/WorkQueue/Policy/Start/Block.py
index a1babc5cd..a8d974f52 100644
--- a/src/python/WMCore/WorkQueue/Policy/Start/Block.py
+++ b/src/python/WMCore/WorkQueue/Policy/Start/Block.py
@@ -232,7 +232,7 @@ class Block(StartPolicyInterface):
         # Fill maskedLumis part of maskedBlocks
 
         for block in maskedBlocks:
-            fileLumis = dbs.dbs.listFileLumis(block_name=block)
+            fileLumis = dbs.dbs.listFileLumis(block_name=block, validFileOnly = 1)
             for fileLumi in fileLumis:
                 lfn = fileLumi['logical_file_name']
                 # For each run : [lumis] mask by needed lumis, append to maskedBlocks
diff --git a/src/python/WMQuality/Emulators/DBSClient/DBS3Reader.py b/src/python/WMQuality/Emulators/DBSClient/DBS3Reader.py
index ce16e818e..2cac54ea5 100644
--- a/src/python/WMQuality/Emulators/DBSClient/DBS3Reader.py
+++ b/src/python/WMQuality/Emulators/DBSClient/DBS3Reader.py
@@ -38,7 +38,7 @@ class _MockDBSApi():
 
         return res
 
-    def listFileLumis(self, logical_file_name = None, block_name = None):
+    def listFileLumis(self, logical_file_name = None, block_name = None, validFileOnly = 1):
         """
         Mock out listFileLumis
         """
@@ -66,7 +66,7 @@ class _MockDBSApi():
 
         return results
 
-    def listFileSummaries(self, block_name = None, dataset = None, run_num = None):
+    def listFileSummaries(self, block_name = None, dataset = None, run_num = None, validFileOnly = 1):
         """
         Mock out listFileSummaries