Repository: dmwm/wmcore
Issue: Add validation for parent flag

1. If input dataset is "RAW" tier, there is no parent files for input files so parent flag cannot be set true. 
2. If there is no input dataset (MC), parent flag cannot be set true.


Changed files:
test/python/WMCore_t/WMSpec_t/StdSpecs_t/StdBase_t.py, src/python/WMCore/WMSpec/WMWorkloadTools.py, src/python/WMCore/WMSpec/StdSpecs/StdBase.py

diff --git a/src/python/WMCore/WMSpec/StdSpecs/StdBase.py b/src/python/WMCore/WMSpec/StdSpecs/StdBase.py
index 16194ef65..57bce4c31 100644
--- a/src/python/WMCore/WMSpec/StdSpecs/StdBase.py
+++ b/src/python/WMCore/WMSpec/StdSpecs/StdBase.py
@@ -12,7 +12,7 @@ from WMCore.Lexicon import lfnBase, identifier, acqname, cmsswversion, cmsname,
 from WMCore.Services.Dashboard.DashboardReporter import DashboardReporter
 from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException
 from WMCore.WMSpec.WMWorkload import newWorkload
-from WMCore.WMSpec.WMWorkloadTools import makeList, makeLumiList, strToBool, checkDBSUrl, validateArgumentsCreate
+from WMCore.WMSpec.WMWorkloadTools import makeList, makeLumiList, strToBool, checkDBSURL, validateArgumentsCreate
 
 
 analysisTaskTypes = ['Analysis', 'PrivateMC']
@@ -908,7 +908,7 @@ class StdBase(object):
                                          "validate" : lambda x : x > 0},
                      "ValidStatus" : {"default" : "PRODUCTION"},
                      "DbsUrl" : {"default" : "https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
-                                 "null" : True, "validate" : checkDBSUrl},
+                                 "null" : True, "validate" : checkDBSURL},
                      "DashboardHost" : {"default" : "cms-wmagent-job.cern.ch"},
                      "DashboardPort" : {"default" : 8884, "type" : int,
                                         "validate" : lambda x : x > 0},
@@ -1020,10 +1020,11 @@ class StdBase(object):
             # Dashboard parameter must be re-defined for test purposes
             if arg == "DashboardHost":
                 schema[arg] = "127.0.0.1"
-            if arg == "CouchURL" or arg == "ConfigCacheUrl":
+            elif arg == "CouchURL" or arg == "ConfigCacheUrl":
+                
                 import os
                 schema[arg] = os.environ["COUCHURL"]
-            if arg == "CouchDBName":
+            elif arg == "CouchDBName":
                 schema[arg] = "reqmgr_config_cache_t"
             elif not workloadDefinition[arg]["optional"]:
                 if workloadDefinition[arg]["type"] == str:
@@ -1033,5 +1034,6 @@ class StdBase(object):
                         schema[arg] = "fake"
                 if workloadDefinition[arg]["type"] == int or workloadDefinition[arg]["type"] == float:
                     schema[arg] = 1
-            
+            else:
+                schema[arg] = workloadDefinition[arg]['default']
         return schema
diff --git a/src/python/WMCore/WMSpec/WMWorkloadTools.py b/src/python/WMCore/WMSpec/WMWorkloadTools.py
index d6ece687f..4be31c569 100644
--- a/src/python/WMCore/WMSpec/WMWorkloadTools.py
+++ b/src/python/WMCore/WMSpec/WMWorkloadTools.py
@@ -57,24 +57,31 @@ def strToBool(string):
         return False
     else:
         raise WMSpecFactoryException("Can't convert to bool: %s" % string)
+
+def _verifyDBSCall(dbsURL, uri):
+    try:
+        #from WMCore.Services.DBS.DBS3Reader import DBS3Reader
+        #DBS3Reader(dbsUrl).dbs.serverinfo()
+        from WMCore.Services.Requests import JSONRequests
+        jsonSender = JSONRequests(dbsURL)
+        result = jsonSender.get("/%s" % uri)
+        if not result[1] == 200:
+            raise WMSpecFactoryException("DBS is not connected: %s : %s" % (dbsURL, str(result)))
+    except:
+        raise WMSpecFactoryException("DBS is not responding: %s" % dbsURL)
+    
+    return result[0]
     
-def checkDBSUrl(dbsUrl):
+def checkDBSURL(dbsURL):
     # use the import statement here since this is packed and used in RunTime code.
     # dbs client is not shipped with it.
     
-    if dbsUrl:
-        try:
-            #from WMCore.Services.DBS.DBS3Reader import DBS3Reader
-            #DBS3Reader(dbsUrl).dbs.serverinfo()
-            from WMCore.Services.Requests import JSONRequests
-            jsonSender = JSONRequests(dbsUrl)
-            result = jsonSender.get("/serverinfo")
-            if not result[1] == 200:
-                raise WMSpecFactoryException("DBS is not connected: %s : %s" % (dbsUrl, str(result)))
-        except:
-            raise WMSpecFactoryException("DBS is not responding: %s" % dbsUrl)
+    if dbsURL:
+        _verifyDBSCall(dbsURL, "serverinfo")
+        return True
     
     return True
+
     
 def parsePileupConfig(mcPileup, dataPileup):
     """
@@ -134,6 +141,32 @@ def _validateArgumentOptions(arguments, argumentDefinition, optionKey):
             arguments[argument] = _validateArgument(argument, arguments[argument], argumentDefinition)
         return
 
+def _validateInputDataset(arguments):
+    inputdataset = arguments.get("InputDataset", None)
+    if inputdataset != None:
+        dbsURL = arguments.get("DbsUrl", None)
+        result = _varifyDBSCall(dbsURL, "datasets?&dataset_access_type=*&dataset=%s" % inputdataset)
+        if len(result) == 0:
+            msg = "Inputdataset %s doesn't exist on %s" % (inputdataset, dbsURL)
+            raise WMSpecFactoryException(msg)
+    return
+
+def validateInputDatasSetAndParentFlag(arguments):
+    inputdataset = arguments.get("InputDataset", None)
+    if arguments.get("IncludeParents", False):
+        if inputdataset == None:
+            msg = "Validation failed: IncludeParent flag is True but there is no inputdataset"
+            raise WMSpecFactoryException(msg)
+        else:
+            dbsURL = arguments.get("DbsUrl", None)
+            result = _varifyDBSCall(dbsURL, "datasetparents?dataset=%s" % inputdataset)
+            if len(result) == 0:
+                msg = "Validation failed: IncludeParent flag is True but inputdataset %s doesn't have parents" % (inputdataset)
+                raise WMSpecFactoryException(msg)
+    else:
+        _validateInputDataset(arguments)
+    return
+            
 def validateArgumentsCreate(arguments, argumentDefinition):
     """
     _validateArguments_
@@ -144,8 +177,10 @@ def validateArgumentsCreate(arguments, argumentDefinition):
     otherwise returns None, this is used for spec creation 
     checks the whether argument is optional as well as validation
     """
-    return _validateArgumentOptions(arguments, argumentDefinition, "optional")
-    
+    _validateArgumentOptions(arguments, argumentDefinition, "optional")
+    validateInputDatasSetAndParentFlag(arguments)
+    return
+
 def validateArgumentsUpdate(arguments, argumentDefinition):
     """
     _validateArgumentsUpdate_
diff --git a/test/python/WMCore_t/WMSpec_t/StdSpecs_t/StdBase_t.py b/test/python/WMCore_t/WMSpec_t/StdSpecs_t/StdBase_t.py
index 796694b57..5ff2edff8 100644
--- a/test/python/WMCore_t/WMSpec_t/StdSpecs_t/StdBase_t.py
+++ b/test/python/WMCore_t/WMSpec_t/StdSpecs_t/StdBase_t.py
@@ -7,8 +7,9 @@ Created on Jun 14, 2013
 """
 
 import unittest
-
+from nose.plugins.attrib import attr
 from WMCore.WMSpec.StdSpecs.StdBase import StdBase
+from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException
 
 class StdBaseTest(unittest.TestCase):
     """
@@ -42,12 +43,38 @@ class StdBaseTest(unittest.TestCase):
         i.e. no exception should be raised.
         """
         arguments = StdBase.getTestArguments()
-        print
         for k in sorted(arguments.keys()):
             print k, arguments[k]
         stdBaseInstance = StdBase()
         stdBaseInstance.factoryWorkloadConstruction("TestWorkload", arguments)
         return
+    
+    @attr('integration')
+    def testStdBaseIncludeParentsValidation(self):
+        """
+        _testStdBaseValidation_
+
+        Check that the test arguments pass basic validation,
+        i.e. no exception should be raised.
+        """
+        arguments = StdBase.getTestArguments()
+        stdBaseInstance = StdBase()
+        
+        arguments["IncludeParents"] = True
+        arguments["InputDataset"] = "/Cosmics/Commissioning2015-v1/RAW"
+        self.assertRaises(WMSpecFactoryException, stdBaseInstance.factoryWorkloadConstruction, "TestWorkload", arguments)
+        
+        arguments["IncludeParents"] = True
+        self.assertRaises(WMSpecFactoryException, stdBaseInstance.factoryWorkloadConstruction, "TestWorkload", arguments)
+        
+        arguments["IncludeParents"] = True
+        arguments["InputDataset"] = "/Cosmics/Commissioning2015-6Mar2015-v1/RECO"
+        stdBaseInstance.factoryWorkloadConstruction("TestWorkload", arguments)
+        
+        arguments["IncludeParents"] = False
+        arguments["InputDataset"] = "/Cosmics/Commissioning2015-6Mar2015-v1/RECO"
+        stdBaseInstance.factoryWorkloadConstruction("TestWorkload", arguments)
+        return 
 
 if __name__ == "__main__":
     unittest.main()