Repository: HECBioSim/Longbow

Issue:
Allow replicate directory naming schemes
At the moment users have to have a specific fixed structure for replicates where the directory consists of rep[x] where the rep part is fixed and the number is incremented. Users have requested that the rep part is flexible so Longbow can be better chained with other tools.

Discussion:

Patch:
diff --git a/longbow/applications.py b/longbow/applications.py
index c693b42..8f99c1e 100755
--- a/longbow/applications.py
+++ b/longbow/applications.py
@@ -353,13 +353,15 @@ def _procfiles(job, arg, filelist, foundflags, substitution):
         # Otherwise we have a replicate job so check these.
         else:
 
-            # Add the repX dir
-            if ("rep" + str(rep)) not in filelist:
+            repx = str(job["replicate-naming"]) + str(rep)
 
-                filelist.append("rep" + str(rep))
+            # Add the repx dir
+            if (repx) not in filelist:
+
+                filelist.append(repx)
 
             fileitem = _procfilesreplicatejobs(
-                app, arg, job["localworkdir"], initargs, rep)
+                app, arg, job["localworkdir"], initargs, repx)
 
             job["executableargs"] = initargs
 
@@ -407,21 +409,21 @@ def _procfilessinglejob(app, arg, cwd):
     return fileitem
 
 
-def _procfilesreplicatejobs(app, arg, cwd, initargs, rep):
+def _procfilesreplicatejobs(app, arg, cwd, initargs, repx):
     """Processor for replicate jobs."""
     fileitem = ""
     tmpitem = ""
 
     # We should check that the replicate directory structure exists.
-    if os.path.isdir(os.path.join(cwd, "rep" + str(rep))) is False:
+    if os.path.isdir(os.path.join(cwd, repx)) is False:
 
-        os.mkdir(os.path.join(cwd, "rep" + str(rep)))
+        os.mkdir(os.path.join(cwd, repx))
 
     # If we have a replicate job then we should check if the file resides
     # within ./rep{i} or if it is a global (common to each replicate) file.
-    if os.path.isfile(os.path.join(cwd, "rep" + str(rep), arg)):
+    if os.path.isfile(os.path.join(cwd, repx, arg)):
 
-        fileitem = os.path.join("rep" + str(rep), arg)
+        fileitem = os.path.join(repx, arg)
 
     # Otherwise do we have a file in cwd
     elif os.path.isfile(os.path.join(cwd, arg)):
@@ -440,7 +442,7 @@ def _procfilesreplicatejobs(app, arg, cwd, initargs, rep):
         try:
 
             tmpitem, _ = getattr(apps, app.lower()).defaultfilename(
-                cwd, os.path.join("rep" + str(rep), arg), "")
+                cwd, os.path.join(repx, arg), "")
 
         except AttributeError:
 
diff --git a/longbow/configuration.py b/longbow/configuration.py
index 2e82db1..b35c5c8 100755
--- a/longbow/configuration.py
+++ b/longbow/configuration.py
@@ -103,6 +103,7 @@ JOBTEMPLATE = {
     "remoteworkdir": "",
     "resource": "",
     "replicates": "1",
+    "replicate-naming": "rep",
     "scheduler": "",
     "scripts": "",
     "slurm-gres": "",


Test patch:
diff --git a/tests/unit/applications/test_procfiles.py b/tests/unit/applications/test_procfiles.py
index 01542ab..a3a27f7 100644
--- a/tests/unit/applications/test_procfiles.py
+++ b/tests/unit/applications/test_procfiles.py
@@ -35,6 +35,7 @@ This testing module contains the tests for the applications module methods.
 """
 
 from longbow.applications import _procfiles
+from longbow.configuration import JOBTEMPLATE
 
 
 def test_procfiles_amber():
@@ -43,16 +44,16 @@ def test_procfiles_amber():
     Test to make sure that the file and flag is picked up for an amber-like
     command-line.
     """
+    job = JOBTEMPLATE.copy()
 
     arg = "coords"
     filelist = []
     foundflags = []
-    job = {
-        "executable": "pmemd.MPI",
-        "replicates": "1",
-        "localworkdir": "tests/standards/jobs/single",
-        "executableargs": ["-i", "input", "-c", "coords", "-p", "topol"]
-    }
+
+    job["executable"] = "pmemd.MPI"
+    job["localworkdir"] = "tests/standards/jobs/single"
+    job["executableargs"] = ["-i", "input", "-c", "coords", "-p", "topol"]
+
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -68,15 +69,15 @@ def test_procfiles_charmm():
     command-line.
     """
 
+    job = JOBTEMPLATE.copy()
+
     arg = "topol"
     filelist = []
     foundflags = []
-    job = {
-        "executable": "charmm",
-        "replicates": "1",
-        "localworkdir": "tests/standards/jobs/single",
-        "executableargs": ["<", "topol"]
-    }
+    job["executable"] = "charmm"
+    job["localworkdir"] = "tests/standards/jobs/single"
+    job["executableargs"] = ["<", "topol"]
+
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -92,15 +93,15 @@ def test_procfiles_gromacs():
     command-line.
     """
 
+    job = JOBTEMPLATE.copy()
+
     arg = "test"
     filelist = []
     foundflags = []
-    job = {
-        "executable": "mdrun_mpi",
-        "replicates": "1",
-        "localworkdir": "tests/standards/jobs/single",
-        "executableargs": ["-deffnm", "test"]
-    }
+    job["executable"] = "mdrun_mpi"
+    job["localworkdir"] = "tests/standards/jobs/single"
+    job["executableargs"] = ["-deffnm", "test"]
+
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -116,15 +117,15 @@ def test_procfiles_namd1():
     command-line.
     """
 
+    job = JOBTEMPLATE.copy()
+
     arg = "input"
     filelist = []
     foundflags = []
-    job = {
-        "executable": "namd2",
-        "replicates": "1",
-        "localworkdir": "tests/standards/jobs/single",
-        "executableargs": ["input"]
-    }
+    job["executable"] = "namd2"
+    job["localworkdir"] = "tests/standards/jobs/single"
+    job["executableargs"] = ["input"]
+
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -140,15 +141,15 @@ def test_procfiles_namd2():
     command-line.
     """
 
+    job = JOBTEMPLATE.copy()
+
     arg = "input"
     filelist = []
     foundflags = []
-    job = {
-        "executable": "namd2",
-        "replicates": "1",
-        "localworkdir": "tests/standards/jobs/single",
-        "executableargs": ["input", ">", "output"]
-    }
+    job["executable"] = "namd2"
+    job["localworkdir"] = "tests/standards/jobs/single"
+    job["executableargs"] = ["input", ">", "output"]
+
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -163,15 +164,16 @@ def test_procfiles_reps1():
     Test for replicate variant.
     """
 
+    job = JOBTEMPLATE.copy()
+
     arg = "coords"
     filelist = []
     foundflags = []
-    job = {
-        "executable": "pmemd.MPI",
-        "replicates": "3",
-        "localworkdir": "tests/standards/jobs/replicate",
-        "executableargs": ["-i", "input", "-c", "coords", "-p", "topol"]
-    }
+    job["executable"] = "pmemd.MPI"
+    job["replicates"] = "3"
+    job["localworkdir"] = "tests/standards/jobs/replicate"
+    job["executableargs"] = ["-i", "input", "-c", "coords", "-p", "topol"]
+
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
@@ -187,15 +189,16 @@ def test_procfiles_reps2():
     Test for replicate variant with global.
     """
 
+    job = JOBTEMPLATE.copy()
+
     arg = "topol"
     filelist = []
     foundflags = []
-    job = {
-        "executable": "pmemd.MPI",
-        "replicates": "3",
-        "localworkdir": "tests/standards/jobs/replicate",
-        "executableargs": ["-i", "input", "-c", "coords", "-p", "topol"]
-    }
+    job["executable"] = "pmemd.MPI"
+    job["replicates"] = "3"
+    job["localworkdir"] = "tests/standards/jobs/replicate"
+    job["executableargs"] = ["-i", "input", "-c", "coords", "-p", "topol"]
+
     substitution = {}
 
     foundflags = _procfiles(job, arg, filelist, foundflags, substitution)
diff --git a/tests/unit/applications/test_procfilesreplicatejobs.py b/tests/unit/applications/test_procfilesreplicatejobs.py
index ca2e0a5..03c6253 100644
--- a/tests/unit/applications/test_procfilesreplicatejobs.py
+++ b/tests/unit/applications/test_procfilesreplicatejobs.py
@@ -57,7 +57,7 @@ def test_procfilesreplicatejobs_t1():
     arg = "input"
     cwd = os.path.join(os.getcwd(), "tests/standards/jobs/replicate")
     initargs = ["-i", "input", "-c", "coords", "-p", "topol"]
-    rep = 1
+    rep = "rep1"
 
     fileitem = _procfilesreplicatejobs(app, arg, cwd, initargs, rep)
 
@@ -75,7 +75,7 @@ def test_procfilesreplicatejobs_t2():
     arg = "topol"
     cwd = os.path.join(os.getcwd(), "tests/standards/jobs/replicate")
     initargs = ["-i", "input", "-c", "coords", "-p", "topol"]
-    rep = 1
+    rep = "rep1"
 
     fileitem = _procfilesreplicatejobs(app, arg, cwd, initargs, rep)
 
@@ -93,7 +93,7 @@ def test_procfilesreplicatejobs_t3():
     arg = "test"
     cwd = os.path.join(os.getcwd(), "tests/standards/jobs/replicate")
     initargs = ["-i", "input", "-c", "coords", "-p", "topol"]
-    rep = 1
+    rep = "rep1"
 
     fileitem = _procfilesreplicatejobs(app, arg, cwd, initargs, rep)
 
@@ -110,7 +110,7 @@ def test_procfilesreplicatejobs_t4():
     arg = "test"
     cwd = os.path.join(os.getcwd(), "tests/standards/jobs/replicate")
     initargs = ["-deffnm", "test"]
-    rep = 2
+    rep = "rep2"
 
     fileitem = _procfilesreplicatejobs(app, arg, cwd, initargs, rep)
 
@@ -128,7 +128,7 @@ def test_procfilesreplicatejobs_t5(m_mkdir):
     arg = "test"
     cwd = os.path.join(os.getcwd(), "tests/standards/jobs/replicate")
     initargs = ["-deffnm", "test"]
-    rep = 4
+    rep = "rep4"
 
     fileitem = _procfilesreplicatejobs(app, arg, cwd, initargs, rep)
 
diff --git a/tests/unit/configuration/test_processconfigsresource.py b/tests/unit/configuration/test_processconfigsresource.py
index 04e5618..4a6bc8f 100644
--- a/tests/unit/configuration/test_processconfigsresource.py
+++ b/tests/unit/configuration/test_processconfigsresource.py
@@ -130,6 +130,7 @@ def test_processconfigsresource1():
             "remoteworkdir": "",
             "resource": "host1",
             "replicates": "1",
+            "replicate-naming": "rep",
             "scheduler": "",
             "user": "",
             "upload-exclude": "",
@@ -236,6 +237,7 @@ def test_processconfigsresource2():
             "remoteworkdir": "",
             "resource": "host2",
             "replicates": "1",
+            "replicate-naming": "rep",
             "scheduler": "",
             "user": "",
             "upload-exclude": "",
@@ -343,6 +345,7 @@ def test_processconfigsresource3():
             "remoteworkdir": "",
             "resource": "host1",
             "replicates": "1",
+            "replicate-naming": "rep",
             "scheduler": "",
             "user": "",
             "upload-exclude": "",
@@ -449,6 +452,7 @@ def test_processconfigsresource4():
             "remoteworkdir": "",
             "resource": "host3",
             "replicates": "1",
+            "replicate-naming": "rep",
             "scheduler": "",
             "user": "",
             "upload-exclude": "",