Repository: HECBioSim/Longbow

Issue:
Using pre-existing queue scripts
Hi James,

I was wondering if it would be possible to include the option of longbow using an already existing queue script instead of creating one on the fly. That way I would no need to struggle with longbow if I want to write non-standard commands in the queue script and I could still easily use it to transfer the files, launching the job, monitoring it and download the results when it is done. It would make my life a lot easier for what we are trying to do at the moment. Do you think that is a feature you would like to add?

Thanks a lot!

Discussion:

Patch:
diff --git a/longbow/applications.py b/longbow/applications.py
index 8f99c1e..c693b42 100755
--- a/longbow/applications.py
+++ b/longbow/applications.py
@@ -353,15 +353,13 @@ def _procfiles(job, arg, filelist, foundflags, substitution):
         # Otherwise we have a replicate job so check these.
         else:
 
-            repx = str(job["replicate-naming"]) + str(rep)
+            # Add the repX dir
+            if ("rep" + str(rep)) not in filelist:
 
-            # Add the repx dir
-            if (repx) not in filelist:
-
-                filelist.append(repx)
+                filelist.append("rep" + str(rep))
 
             fileitem = _procfilesreplicatejobs(
-                app, arg, job["localworkdir"], initargs, repx)
+                app, arg, job["localworkdir"], initargs, rep)
 
             job["executableargs"] = initargs
 
@@ -409,21 +407,21 @@ def _procfilessinglejob(app, arg, cwd):
     return fileitem
 
 
-def _procfilesreplicatejobs(app, arg, cwd, initargs, repx):
+def _procfilesreplicatejobs(app, arg, cwd, initargs, rep):
     """Processor for replicate jobs."""
     fileitem = ""
     tmpitem = ""
 
     # We should check that the replicate directory structure exists.
-    if os.path.isdir(os.path.join(cwd, repx)) is False:
+    if os.path.isdir(os.path.join(cwd, "rep" + str(rep))) is False:
 
-        os.mkdir(os.path.join(cwd, repx))
+        os.mkdir(os.path.join(cwd, "rep" + str(rep)))
 
     # If we have a replicate job then we should check if the file resides
     # within ./rep{i} or if it is a global (common to each replicate) file.
-    if os.path.isfile(os.path.join(cwd, repx, arg)):
+    if os.path.isfile(os.path.join(cwd, "rep" + str(rep), arg)):
 
-        fileitem = os.path.join(repx, arg)
+        fileitem = os.path.join("rep" + str(rep), arg)
 
     # Otherwise do we have a file in cwd
     elif os.path.isfile(os.path.join(cwd, arg)):
@@ -442,7 +440,7 @@ def _procfilesreplicatejobs(app, arg, cwd, initargs, repx):
         try:
 
             tmpitem, _ = getattr(apps, app.lower()).defaultfilename(
-                cwd, os.path.join(repx, arg), "")
+                cwd, os.path.join("rep" + str(rep), arg), "")
 
         except AttributeError:
 
diff --git a/longbow/configuration.py b/longbow/configuration.py
index b35c5c8..fde6c01 100755
--- a/longbow/configuration.py
+++ b/longbow/configuration.py
@@ -103,13 +103,13 @@ JOBTEMPLATE = {
     "remoteworkdir": "",
     "resource": "",
     "replicates": "1",
-    "replicate-naming": "rep",
     "scheduler": "",
     "scripts": "",
     "slurm-gres": "",
     "staging-frequency": "300",
     "sge-peflag": "mpi",
     "sge-peoverride": "false",
+    "subfile": "",
     "user": "",
     "upload-exclude": "",
     "upload-include": ""
diff --git a/longbow/scheduling.py b/longbow/scheduling.py
index 1917d77..93c7f2d 100755
--- a/longbow/scheduling.py
+++ b/longbow/scheduling.py
@@ -317,11 +317,18 @@ def prepare(jobs):
 
         try:
 
-            LOG.info("Creating submit file for job '%s'", item)
+            if job["subfile"] == "":
 
-            getattr(schedulers, scheduler.lower()).prepare(job)
+                LOG.info("Creating submit file for job '%s'", item)
 
-            LOG.info("Submit file created successfully")
+                getattr(schedulers, scheduler.lower()).prepare(job)
+
+                LOG.info("Submit file created successfully")
+
+            else:
+
+                LOG.info("For job '%s' user has supplied their own job submit "
+                         "script - skipping creation.", item)
 
         except AttributeError:
 


Test patch:
diff --git a/tests/unit/applications/test_procfiles.py b/tests/unit/applications/test_procfiles.py
index a3a27f7..01542ab 100644
--- a/tests/unit/applications/test_procfiles.py
+++ b/tests/unit/applications/test_procfiles.py
@@ -35,7 +35,6 @@ This testing module contains the tests for the applications module methods.
 """
 
 from longbow.applications import _procfiles
-from longbow.configuration import JOBTEMPLATE
 
 
 def test_procfiles_amber():
@@ -44,16 +43,16 @@ def test_procfiles_amber():
     Test to make sure that the file and flag is picked up for an amber-like
     command-line.
     """
-    job = JOBTEMPLATE.copy()
 
     arg = "coords"
     filelist = []
     foundflags = []
-
-    job["executable"] = "pmemd.MPI"
-    job["localworkdir"] = "tests/standards/jobs/single"
-    job["executableargs"] = ["-i", "input", "-c", "coords", "-p", "topol"]
-
+    job = {
+        "executable": "pmemd.MPI",
+        "replicates": "1",
+        "localworkdir": "tests/standards/jobs/single",
+        "executableargs": ["-i", "input", "-c", "coords", "-p", "topol"]
+    }
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -69,15 +68,15 @@ def test_procfiles_charmm():
     command-line.
     """
 
-    job = JOBTEMPLATE.copy()
-
     arg = "topol"
     filelist = []
     foundflags = []
-    job["executable"] = "charmm"
-    job["localworkdir"] = "tests/standards/jobs/single"
-    job["executableargs"] = ["<", "topol"]
-
+    job = {
+        "executable": "charmm",
+        "replicates": "1",
+        "localworkdir": "tests/standards/jobs/single",
+        "executableargs": ["<", "topol"]
+    }
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -93,15 +92,15 @@ def test_procfiles_gromacs():
     command-line.
     """
 
-    job = JOBTEMPLATE.copy()
-
     arg = "test"
     filelist = []
     foundflags = []
-    job["executable"] = "mdrun_mpi"
-    job["localworkdir"] = "tests/standards/jobs/single"
-    job["executableargs"] = ["-deffnm", "test"]
-
+    job = {
+        "executable": "mdrun_mpi",
+        "replicates": "1",
+        "localworkdir": "tests/standards/jobs/single",
+        "executableargs": ["-deffnm", "test"]
+    }
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -117,15 +116,15 @@ def test_procfiles_namd1():
     command-line.
     """
 
-    job = JOBTEMPLATE.copy()
-
     arg = "input"
     filelist = []
     foundflags = []
-    job["executable"] = "namd2"
-    job["localworkdir"] = "tests/standards/jobs/single"
-    job["executableargs"] = ["input"]
-
+    job = {
+        "executable": "namd2",
+        "replicates": "1",
+        "localworkdir": "tests/standards/jobs/single",
+        "executableargs": ["input"]
+    }
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -141,15 +140,15 @@ def test_procfiles_namd2():
     command-line.
     """
 
-    job = JOBTEMPLATE.copy()
-
     arg = "input"
     filelist = []
     foundflags = []
-    job["executable"] = "namd2"
-    job["localworkdir"] = "tests/standards/jobs/single"
-    job["executableargs"] = ["input", ">", "output"]
-
+    job = {
+        "executable": "namd2",
+        "replicates": "1",
+        "localworkdir": "tests/standards/jobs/single",
+        "executableargs": ["input", ">", "output"]
+    }
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -164,16 +163,15 @@ def test_procfiles_reps1():
     Test for replicate variant.
     """
 
-    job = JOBTEMPLATE.copy()
-
     arg = "coords"
     filelist = []
     foundflags = []
-    job["executable"] = "pmemd.MPI"
-    job["replicates"] = "3"
-    job["localworkdir"] = "tests/standards/jobs/replicate"
-    job["executableargs"] = ["-i", "input", "-c", "coords", "-p", "topol"]
-
+    job = {
+        "executable": "pmemd.MPI",
+        "replicates": "3",
+        "localworkdir": "tests/standards/jobs/replicate",
+        "executableargs": ["-i", "input", "-c", "coords", "-p", "topol"]
+    }
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -189,16 +187,15 @@ def test_procfiles_reps2():
     Test for replicate variant with global.
     """
 
-    job = JOBTEMPLATE.copy()
-
     arg = "topol"
     filelist = []
     foundflags = []
-    job["executable"] = "pmemd.MPI"
-    job["replicates"] = "3"
-    job["localworkdir"] = "tests/standards/jobs/replicate"
-    job["executableargs"] = ["-i", "input", "-c", "coords", "-p", "topol"]
-
+    job = {
+        "executable": "pmemd.MPI",
+        "replicates": "3",
+        "localworkdir": "tests/standards/jobs/replicate",
+        "executableargs": ["-i", "input", "-c", "coords", "-p", "topol"]
+    }
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
diff --git a/tests/unit/applications/test_procfilesreplicatejobs.py b/tests/unit/applications/test_procfilesreplicatejobs.py
index 03c6253..ca2e0a5 100644
--- a/tests/unit/applications/test_procfilesreplicatejobs.py
+++ b/tests/unit/applications/test_procfilesreplicatejobs.py
@@ -57,7 +57,7 @@ def test_procfilesreplicatejobs_t1():
     arg = "input"
     cwd = os.path.join(os.getcwd(), "tests/standards/jobs/replicate")
     initargs = ["-i", "input", "-c", "coords", "-p", "topol"]
-    rep = "rep1"
+    rep = 1
 
     fileitem = _procfilesreplicatejobs(app, arg, cwd, initargs, rep)
 
@@ -75,7 +75,7 @@ def test_procfilesreplicatejobs_t2():
     arg = "topol"
     cwd = os.path.join(os.getcwd(), "tests/standards/jobs/replicate")
     initargs = ["-i", "input", "-c", "coords", "-p", "topol"]
-    rep = "rep1"
+    rep = 1
 
     fileitem = _procfilesreplicatejobs(app, arg, cwd, initargs, rep)
 
@@ -93,7 +93,7 @@ def test_procfilesreplicatejobs_t3():
     arg = "test"
     cwd = os.path.join(os.getcwd(), "tests/standards/jobs/replicate")
     initargs = ["-i", "input", "-c", "coords", "-p", "topol"]
-    rep = "rep1"
+    rep = 1
 
     fileitem = _procfilesreplicatejobs(app, arg, cwd, initargs, rep)
 
@@ -110,7 +110,7 @@ def test_procfilesreplicatejobs_t4():
     arg = "test"
     cwd = os.path.join(os.getcwd(), "tests/standards/jobs/replicate")
     initargs = ["-deffnm", "test"]
-    rep = "rep2"
+    rep = 2
 
     fileitem = _procfilesreplicatejobs(app, arg, cwd, initargs, rep)
 
@@ -128,7 +128,7 @@ def test_procfilesreplicatejobs_t5(m_mkdir):
     arg = "test"
     cwd = os.path.join(os.getcwd(), "tests/standards/jobs/replicate")
     initargs = ["-deffnm", "test"]
-    rep = "rep4"
+    rep = 4
 
     fileitem = _procfilesreplicatejobs(app, arg, cwd, initargs, rep)
 
diff --git a/tests/unit/configuration/test_processconfigsresource.py b/tests/unit/configuration/test_processconfigsresource.py
index 4a6bc8f..05423a2 100644
--- a/tests/unit/configuration/test_processconfigsresource.py
+++ b/tests/unit/configuration/test_processconfigsresource.py
@@ -130,8 +130,8 @@ def test_processconfigsresource1():
             "remoteworkdir": "",
             "resource": "host1",
             "replicates": "1",
-            "replicate-naming": "rep",
             "scheduler": "",
+            "subfile": "",
             "user": "",
             "upload-exclude": "",
             "upload-include": ""
@@ -237,8 +237,8 @@ def test_processconfigsresource2():
             "remoteworkdir": "",
             "resource": "host2",
             "replicates": "1",
-            "replicate-naming": "rep",
             "scheduler": "",
+            "subfile": "",
             "user": "",
             "upload-exclude": "",
             "upload-include": ""
@@ -345,8 +345,8 @@ def test_processconfigsresource3():
             "remoteworkdir": "",
             "resource": "host1",
             "replicates": "1",
-            "replicate-naming": "rep",
             "scheduler": "",
+            "subfile": "",
             "user": "",
             "upload-exclude": "",
             "upload-include": ""
@@ -452,8 +452,8 @@ def test_processconfigsresource4():
             "remoteworkdir": "",
             "resource": "host3",
             "replicates": "1",
-            "replicate-naming": "rep",
             "scheduler": "",
+            "subfile": "",
             "user": "",
             "upload-exclude": "",
             "upload-include": ""
diff --git a/tests/unit/scheduling/test_prepare.py b/tests/unit/scheduling/test_prepare.py
index 1e5be06..dff69d5 100644
--- a/tests/unit/scheduling/test_prepare.py
+++ b/tests/unit/scheduling/test_prepare.py
@@ -60,7 +60,8 @@ def test_prepare_single(mock_prepare):
         "job-one": {
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test456"
+            "jobid": "test456",
+            "subfile": ""
         }
     }
 
@@ -82,17 +83,20 @@ def test_prepare_multiple(mock_prepare):
         "job-one": {
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test123"
+            "jobid": "test123",
+            "subfile": ""
         },
         "job-two": {
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test456"
+            "jobid": "test456",
+            "subfile": ""
         },
         "job-three": {
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test789"
+            "jobid": "test789",
+            "subfile": ""
         }
     }
 
@@ -113,7 +117,8 @@ def test_prepare_attrexcept(mock_prepare):
         "job-one": {
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test456"
+            "jobid": "test456",
+            "subfile": ""
         }
     }
 
@@ -122,3 +127,25 @@ def test_prepare_attrexcept(mock_prepare):
     with pytest.raises(exceptions.PluginattributeError):
 
         prepare(jobs)
+
+
+@mock.patch('longbow.schedulers.lsf.prepare')
+def test_prepare_ownscript(mock_prepare):
+
+    """
+    Test that if user supplies a script that longbow doesn't create one.
+    """
+
+    jobs = {
+        "job-one": {
+            "resource": "test-machine",
+            "scheduler": "LSF",
+            "jobid": "test456",
+            "subfile": "test.lsf"
+        }
+    }
+
+    prepare(jobs)
+
+    assert mock_prepare.call_count == 0, \
+        "This method shouldn't be called at all in this case."