Repository: DataBiosphere/toil

Issue:
Option --restart produces error when running CWL pipeline
``` bash
> cwltoil --restart --jobStore file:jobStore ppeline.cwl job.yml
pc: 2016-08-02 10:24:27,230 INFO: toil.lib.bioio: Logging set at level: INFO
pc: 2016-08-02 10:24:44,319 INFO: toil.lib.bioio: Logging set at level: INFO
pc: 2016-08-02 10:24:44,654 INFO: toil.jobStores.fileJobStore: Path to job store directory is '/data/cwl-test/jobStore'.
Traceback (most recent call last):
  File "/data/cwl-test/toil/bin/cwltoil", line 11, in <module>
    sys.exit(main())
  File "/data/cwl-test/toil/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 595, in main
    outobj = toil.start(wf1)
  File "/data/cwl-test/toil/lib/python2.7/site-packages/toil/common.py", line 536, in start
    raise ToilRestartException('A Toil workflow can only be started once. Use '
toil.common.ToilRestartException: A Toil workflow can only be started once. Use Toil.restart() to resume it.
```

Is the restart feature implemented for CWL?

Thanks!

Option --restart produces error when running CWL pipeline
``` bash
> cwltoil --restart --jobStore file:jobStore ppeline.cwl job.yml
pc: 2016-08-02 10:24:27,230 INFO: toil.lib.bioio: Logging set at level: INFO
pc: 2016-08-02 10:24:44,319 INFO: toil.lib.bioio: Logging set at level: INFO
pc: 2016-08-02 10:24:44,654 INFO: toil.jobStores.fileJobStore: Path to job store directory is '/data/cwl-test/jobStore'.
Traceback (most recent call last):
  File "/data/cwl-test/toil/bin/cwltoil", line 11, in <module>
    sys.exit(main())
  File "/data/cwl-test/toil/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 595, in main
    outobj = toil.start(wf1)
  File "/data/cwl-test/toil/lib/python2.7/site-packages/toil/common.py", line 536, in start
    raise ToilRestartException('A Toil workflow can only be started once. Use '
toil.common.ToilRestartException: A Toil workflow can only be started once. Use Toil.restart() to resume it.
```

Is the restart feature implemented for CWL?

Thanks!



Discussion:
Apologies, `--restart` is not yet implemented for cwltoil but this is something we're actively working on. You can see the discussion in #1025 For the short term you'll have to remove the previous jobStore directory and re-run from the start.

Great to see that you guys are already working on this. And thanks for pointing me to the pull request discussion, I somehow overlooked it.

Apologies, `--restart` is not yet implemented for cwltoil but this is something we're actively working on. You can see the discussion in #1025 For the short term you'll have to remove the previous jobStore directory and re-run from the start.

Great to see that you guys are already working on this. And thanks for pointing me to the pull request discussion, I somehow overlooked it.


Resolving patch:
diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py
--- a/src/toil/cwl/cwltoil.py
+++ b/src/toil/cwl/cwltoil.py
@@ -649,20 +649,23 @@ def importDefault(tool):
             adjustFiles(tool, functools.partial(writeFile, toil.importFile, {}))
         t.visit(importDefault)
 
-        basedir = os.path.dirname(os.path.abspath(options.cwljob or options.cwltool))
-        builder = t._init_job(job, basedir=basedir)
-        (wf1, wf2) = makeJob(t, {}, use_container=use_container, preserve_environment=options.preserve_environment, tmpdir=os.path.realpath(outdir))
-        cwltool.builder.adjustDirObjs(builder.job, locToPath)
-        cwltool.builder.adjustFileObjs(builder.job, locToPath)
-        adjustFiles(builder.job, lambda x: "file://%s" % os.path.abspath(os.path.join(basedir, x))
-                    if not urlparse.urlparse(x).scheme else x)
-        cwltool.builder.adjustDirObjs(builder.job, pathToLoc)
-        cwltool.builder.adjustFileObjs(builder.job, pathToLoc)
-        cwltool.builder.adjustFileObjs(builder.job, addFilePartRefs)
-        adjustFiles(builder.job, functools.partial(writeFile, toil.importFile, {}))
-        wf1.cwljob = builder.job
-
-        outobj = toil.start(wf1)
+        if options.restart:
+            outobj = toil.restart()
+        else:
+            basedir = os.path.dirname(os.path.abspath(options.cwljob or options.cwltool))
+            builder = t._init_job(job, basedir=basedir)
+            (wf1, wf2) = makeJob(t, {}, use_container=use_container, preserve_environment=options.preserve_environment, tmpdir=os.path.realpath(outdir))
+            cwltool.builder.adjustDirObjs(builder.job, locToPath)
+            cwltool.builder.adjustFileObjs(builder.job, locToPath)
+            adjustFiles(builder.job, lambda x: "file://%s" % os.path.abspath(os.path.join(basedir, x))
+                        if not urlparse.urlparse(x).scheme else x)
+            cwltool.builder.adjustDirObjs(builder.job, pathToLoc)
+            cwltool.builder.adjustFileObjs(builder.job, pathToLoc)
+            cwltool.builder.adjustFileObjs(builder.job, addFilePartRefs)
+            adjustFiles(builder.job, functools.partial(writeFile, toil.importFile, {}))
+            wf1.cwljob = builder.job
+            outobj = toil.start(wf1)
+
         outobj = resolve_indirect(outobj)
 
         adjustFilesWithSecondary(outobj, functools.partial(getFile, toil, outdir, index={}, export=True, rename_collision=True))


Test patch:
diff --git a/src/toil/test/cwl/cwlTest.py b/src/toil/test/cwl/cwlTest.py
--- a/src/toil/test/cwl/cwlTest.py
+++ b/src/toil/test/cwl/cwlTest.py
@@ -50,6 +50,36 @@ def test_run_revsort(self):
                 u'class': u'File',
                 u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})
 
+    def test_restart(self):
+        """Enable restarts with CWLtoil -- run failing test, re-run correct test.
+        """
+        from toil.cwl import cwltoil
+        from toil.jobStores.abstractJobStore import NoSuchJobStoreException
+        from toil.leader import FailedJobsException
+        outDir = self._createTempDir()
+        cwlDir = os.path.join(self._projectRootPath(), "src", "toil", "test", "cwl")
+        cmd = ['--outdir', outDir, '--jobStore', os.path.join(outDir, 'jobStore'), "--no-container",
+               os.path.join(cwlDir, "revsort.cwl"), os.path.join(cwlDir, "revsort-job.json")]
+        def path_without_rev():
+            return ":".join([d for d in os.environ["PATH"].split(":")
+                             if not os.path.exists(os.path.join(d, "rev"))])
+        orig_path = os.environ["PATH"]
+        # Force a failure and half finished job by removing `rev` from the PATH
+        os.environ["PATH"] = path_without_rev()
+        try:
+            cwltoil.main(cmd)
+            self.fail("Expected problem job with incorrect PATH did not fail")
+        except FailedJobsException:
+            pass
+        # Finish the job with a correct PATH
+        os.environ["PATH"] = orig_path
+        cwltoil.main(cmd + ["--restart"])
+        # Should fail because previous job completed successfully
+        try:
+            cwltoil.main(cmd + ["--restart"])
+            self.fail("Restart with missing directory did not fail")
+        except NoSuchJobStoreException:
+            pass
 
     def test_run_conformance(self):
         rootDir = self._projectRootPath()