Repository: Duke-GCB/lando

Issue:
cwlreport uses badly formed path for packed workflows
Test job 70 with a packed workflow failed because `workflow_to_report` included `#main`

```
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/lando/worker/worker.py", line 186, in run
    self.func(working_directory, self.payload)
  File "/usr/local/lib/python3.5/dist-packages/lando/worker/worker.py", line 103, in run_workflow
    workflow.run(payload.workflow_type, payload.workflow_url, payload.workflow_path, payload.job_order)
  File "/usr/local/lib/python3.5/dist-packages/lando/worker/cwlworkflow.py", line 243, in run
    results_directory.add_files(process)
  File "/usr/local/lib/python3.5/dist-packages/lando/worker/cwlworkflow.py", line 359, in add_files
    self._create_report(cwl_process)
  File "/usr/local/lib/python3.5/dist-packages/lando/worker/cwlworkflow.py", line 392, in _create_report
    workflow_info = create_workflow_info(workflow_path=os.path.join(workflow_directory, self.workflow_downloader.workflow_to_report))
  File "/usr/local/lib/python3.5/dist-packages/lando/worker/cwlreport.py", line 114, in create_workflow_info
    doc = parse_yaml_or_json(workflow_path)
  File "/usr/local/lib/python3.5/dist-packages/lando/worker/cwlreport.py", line 313, in parse_yaml_or_json
    with codecs.open(path, mode='r', encoding='utf-8') as infile:
  File "/usr/lib/python3.5/codecs.py", line 895, in open
    file = builtins.open(filename, mode, buffering)
FileNotFoundError: [Errno 2] No such file or directory: 'data_for_job_70/working/results/docs/scripts/exomeseq.cwl#main'
```

Discussion:

Patch:
diff --git a/lando/worker/cwlworkflow.py b/lando/worker/cwlworkflow.py
index d54b2f5..a81f3f8 100644
--- a/lando/worker/cwlworkflow.py
+++ b/lando/worker/cwlworkflow.py
@@ -97,8 +97,10 @@ class CwlWorkflowDownloader(object):
         self.workflow_url = workflow_url
         self.workflow_path = workflow_path
         self.workflow_basename = os.path.basename(self.workflow_url)
-        self.workflow_to_run = None # set by handle_*_download
-        self.workflow_to_report = None
+        # below are set by handle_*_download
+        self.workflow_to_run = None # Absolute path of workflow in lando VM
+        self.workflow_to_report = None # Relative path to use in templated report (may include #main)
+        self.workflow_to_read = None # Relative path to use when parsing for workflow inputs
         self.download_path = os.path.join(self.working_directory, self.workflow_basename)
         urllib.request.urlretrieve(self.workflow_url, self.download_path)
         if self.workflow_type == self.TYPE_PACKED:
@@ -112,6 +114,7 @@ class CwlWorkflowDownloader(object):
         # After downloading packed workflow, just append the workflow path to the local file name
         self.workflow_to_run = self.download_path + self.workflow_path
         self.workflow_to_report = self.workflow_basename + self.workflow_path
+        self.workflow_to_read = self.workflow_basename
 
     def _unzip_to_directory(self, directory):
         with zipfile.ZipFile(self.download_path) as z:
@@ -122,6 +125,7 @@ class CwlWorkflowDownloader(object):
         self._unzip_to_directory(self.working_directory)
         self.workflow_to_run = os.path.join(self.working_directory, self.workflow_path)
         self.workflow_to_report = self.workflow_path
+        self.workflow_to_read = self.workflow_path
 
     def copy_to_directory(self, directory):
         """
@@ -389,7 +393,7 @@ class ResultsDirectory(object):
         """
         logs_directory = os.path.join(self.docs_directory, LOGS_DIRECTORY)
         workflow_directory = os.path.join(self.docs_directory, WORKFLOW_DIRECTORY)
-        workflow_info = create_workflow_info(workflow_path=os.path.join(workflow_directory, self.workflow_downloader.workflow_to_report))
+        workflow_info = create_workflow_info(workflow_path=os.path.join(workflow_directory, self.workflow_downloader.workflow_to_read))
         workflow_info.update_with_job_order(job_order_path=os.path.join(workflow_directory, self.job_order_filename))
         workflow_info.update_with_job_output(job_output_path=os.path.join(logs_directory, JOB_STDOUT_FILENAME))
         job_data = {


Test patch:
diff --git a/lando/worker/tests/test_cwlworkflow.py b/lando/worker/tests/test_cwlworkflow.py
index 4b67f7c..0fa7fd0 100644
--- a/lando/worker/tests/test_cwlworkflow.py
+++ b/lando/worker/tests/test_cwlworkflow.py
@@ -352,9 +352,10 @@ class TestResultsDirectory(TestCase):
                        mock_save_data_to_directory, mock_create_dir_if_necessary):
         job_id = 1
         workflow_downloader = create_autospec(CwlWorkflowDownloader,
-                                              workflow_to_run='/tmp/nosuchpath.cwl',
-                                              workflow_to_report='nosuch.cwl',
-                                              workflow_basename='nosuch.cwl')
+                                              workflow_to_run='/tmp/nosuch-run.cwl',
+                                              workflow_to_report='nosuch-report.cwl',
+                                              workflow_basename='nosuch-base.cwl',
+                                              workflow_to_read='nosuch-read.cwl')
         cwl_directory = create_autospec(CwlDirectory,
                                         result_directory='/tmp/fakedir',
                                         job_order_file_path='/tmp/alsonotreal.json',
@@ -400,7 +401,7 @@ class TestResultsDirectory(TestCase):
             call(documentation_directory + 'scripts')
         ])
         mock_create_workflow_info.assert_has_calls([
-            call(workflow_path=(documentation_directory + 'scripts/nosuch.cwl')),
+            call(workflow_path=(documentation_directory + 'scripts/nosuch-read.cwl')),
             call().update_with_job_order(job_order_path=(documentation_directory + 'scripts/alsonotreal.json')),
             call().update_with_job_output(job_output_path=(documentation_directory + 'logs/cwltool-output.json')),
             call().count_output_files(),
@@ -417,9 +418,10 @@ class TestResultsDirectory(TestCase):
                                              mock_shutil, mock_save_data_to_directory, mock_create_dir_if_necessary):
         job_id = 1
         workflow_downloader = create_autospec(CwlWorkflowDownloader,
-                                              workflow_to_run='/tmp/nosuchpath.cwl',
-                                              workflow_to_report='nosuch.cwl',
-                                              workflow_basename='nosuch.cwl')
+                                              workflow_to_run='/tmp/nosuch.cwl#main',
+                                              workflow_to_report='nosuch-report.cwl',
+                                              workflow_basename='nosuch-base.cwl',
+                                              workflow_to_read='nosuch-read.cwl')
         cwl_directory = create_autospec(CwlDirectory,
                                         result_directory='/tmp/fakedir',
                                         job_order_file_path='/tmp/alsonotreal.json',