Repository: Duke-GCB/lando

Issue:
k8s workflow runner logs are empty
We started redirecting stderr to a file so we can save this long term for successful jobs:
https://github.com/Duke-GCB/lando/blob/6f370eee825f78b694df3a640bc1cb75b4e61ec5/lando/k8s/jobmanager.py#L170-L175
This makes failed jobs have not logs to be saved. The error is recorded in a pvc, but it should also be `tee`d into the k8s output logs as well.

Discussion:

Patch:
diff --git a/lando/k8s/jobmanager.py b/lando/k8s/jobmanager.py
index 438eb1b..70c43e4 100644
--- a/lando/k8s/jobmanager.py
+++ b/lando/k8s/jobmanager.py
@@ -168,23 +168,21 @@ class JobManager(object):
                 mount_path=system_data_volume.mount_path,
                 volume_claim_name=system_data_volume.volume_claim_name,
                 read_only=True))
-        command_parts = run_workflow_config.command
-        command_parts.extend(["--tmp-outdir-prefix", Paths.TMPOUT_DATA + "/",
-                              "--outdir", Paths.OUTPUT_RESULTS_DIR + "/",
-                              "--max-ram", self.job.job_flavor_memory,
-                              "--max-cores", str(self.job.job_flavor_cpus),
-                              "--usage-report", str(self.names.usage_report_path),
-                              ])
-        command_parts.extend([
-            self.names.workflow_path,
-            self.names.job_order_path,
-            ">{}".format(self.names.run_workflow_stdout_path),
-            "2>{}".format(self.names.run_workflow_stderr_path),
-        ])
+        command = run_workflow_config.command
+        command.extend(["--tmp-outdir-prefix", Paths.TMPOUT_DATA + "/",
+                        "--outdir", Paths.OUTPUT_RESULTS_DIR + "/",
+                        "--max-ram", self.job.job_flavor_memory,
+                        "--max-cores", str(self.job.job_flavor_cpus),
+                        "--usage-report", self.names.usage_report_path,
+                        "--stdout", self.names.run_workflow_stdout_path,
+                        "--stderr", self.names.run_workflow_stderr_path,
+                        self.names.workflow_path,
+                        self.names.job_order_path,
+                        ])
         container = Container(
             name=self.names.run_workflow,
             image_name=run_workflow_config.image_name,
-            command=["bash", "-c", ' '.join(command_parts)],
+            command=command,
             env_dict={
                 "CALRISSIAN_POD_NAME": FieldRefEnvVar(field_path="metadata.name")
             },


Test patch:
diff --git a/lando/k8s/tests/test_jobmanager.py b/lando/k8s/tests/test_jobmanager.py
index 7a145a5..dd8640b 100644
--- a/lando/k8s/tests/test_jobmanager.py
+++ b/lando/k8s/tests/test_jobmanager.py
@@ -198,11 +198,11 @@ class TestJobManager(TestCase):
                                 '--outdir /bespin/output-data/results/ ' \
                                 '--max-ram 1G --max-cores 2 ' \
                                 '--usage-report /bespin/output-data/job-51-jpb-resource-usage.json ' \
+                                '--stdout /bespin/output-data/bespin-workflow-output.json ' \
+                                '--stderr /bespin/output-data/bespin-workflow-output.log ' \
                                 '/bespin/job-data/workflow/someurl ' \
-                                '/bespin/job-data/job-order.json ' \
-                                '>/bespin/output-data/bespin-workflow-output.json ' \
-                                '2>/bespin/output-data/bespin-workflow-output.log'
-        self.assertEqual(job_container.command, ['bash', '-c', expected_bash_command],
+                                '/bespin/job-data/job-order.json'.split(' ')
+        self.assertEqual(job_container.command, expected_bash_command,
                          'run workflow command combines job settings and staged files')
         self.assertEqual(job_container.env_dict['CALRISSIAN_POD_NAME'].field_path, 'metadata.name',
                          'We should store the pod name in a CALRISSIAN_POD_NAME environment variable')