Repository: Parsl/parsl

Issue:
LocalProvider+LocalChannel discards stdout/stderr of launched command
**Describe the bug**
LocalProvider+LocalChannel directs stdout/stderr of the submitted command into a pipe that is then not used.

This means that stdout/err from the command is lost.

**To Reproduce**
Run HTEX with LocalProvider+LocalChannel+SingleNodeLauncher. The "Found cores :" message from SingleNodeLauncher does not appear in a log file.

**Expected behavior**
This output should not be discarded. For example, with slurm, the output of the executed batch job goes into a log file alongside the batch job submit file.

**Environment**
parsl master ea8333c3249fdf3b21691b7a66fb4e9c8ed37ec0


Discussion:
the plan in #1448 should resolve this, as the non-LocalChannel code path directs output to a file

Resolving patch:
diff --git a/parsl/providers/ad_hoc/ad_hoc.py b/parsl/providers/ad_hoc/ad_hoc.py
--- a/parsl/providers/ad_hoc/ad_hoc.py
+++ b/parsl/providers/ad_hoc/ad_hoc.py
@@ -166,7 +166,6 @@ def submit(self, command, tasks_per_node, job_name="parsl.adhoc"):
         self._write_submit_script(wrap_command, script_path)
 
         job_id = None
-        proc = None
         remote_pid = None
         final_cmd = None
 
@@ -174,30 +173,21 @@ def submit(self, command, tasks_per_node, job_name="parsl.adhoc"):
             logger.debug("Pushing start script")
             script_path = channel.push_file(script_path, channel.script_dir)
 
-        if not isinstance(channel, LocalChannel):
-            # Bash would return until the streams are closed. So we redirect to a outs file
-            final_cmd = 'bash {0} > {0}.out 2>&1 & \n echo "PID:$!" '.format(script_path)
-            retcode, stdout, stderr = channel.execute_wait(final_cmd, self.cmd_timeout)
-            for line in stdout.split('\n'):
-                if line.startswith("PID:"):
-                    remote_pid = line.split("PID:")[1].strip()
-                    job_id = remote_pid
-            if job_id is None:
-                logger.warning("Channel failed to start remote command/retrieve PID")
-        else:
-            try:
-                final_cmd = 'bash {0}'.format(script_path)
-                job_id, proc = channel.execute_no_wait(final_cmd, self.cmd_timeout)
-            except Exception as e:
-                logger.debug("Channel execute failed for: {}, {}".format(channel, e))
-                raise
+        # Bash would return until the streams are closed. So we redirect to a outs file
+        final_cmd = 'bash {0} > {0}.out 2>&1 & \n echo "PID:$!" '.format(script_path)
+        retcode, stdout, stderr = channel.execute_wait(final_cmd, self.cmd_timeout)
+        for line in stdout.split('\n'):
+            if line.startswith("PID:"):
+                remote_pid = line.split("PID:")[1].strip()
+                job_id = remote_pid
+        if job_id is None:
+            logger.warning("Channel failed to start remote command/retrieve PID")
 
         self.resources[job_id] = {'job_id': job_id,
                                   'status': JobStatus(JobState.RUNNING),
                                   'cmd': final_cmd,
                                   'channel': channel,
-                                  'remote_pid': remote_pid,
-                                  'proc': proc}
+                                  'remote_pid': remote_pid}
 
         return job_id
 
diff --git a/parsl/providers/local/local.py b/parsl/providers/local/local.py
--- a/parsl/providers/local/local.py
+++ b/parsl/providers/local/local.py
@@ -1,6 +1,5 @@
 import logging
 import os
-import signal
 import time
 
 from parsl.channels import LocalChannel
@@ -77,32 +76,16 @@ def status(self, job_ids):
 
         logger.debug("Checking status of: {0}".format(job_ids))
         for job_id in self.resources:
-            if self.resources[job_id]['proc']:
-
-                poll_code = self.resources[job_id]['proc'].poll()
-                if self.resources[job_id]['status'].terminal:
-                    continue
-
-                if poll_code is None:
-                    self.resources[job_id]['status'] = JobStatus(JobState.RUNNING)
-                elif poll_code == 0:
-                    self.resources[job_id]['status'] = JobStatus(JobState.COMPLETED)
-                elif poll_code != 0:
-                    self.resources[job_id]['status'] = JobStatus(JobState.FAILED)
-                else:
-                    logger.error("Internal consistency error: unexpected case in local provider state machine")
-
-            elif self.resources[job_id]['remote_pid']:
-
-                retcode, stdout, stderr = self.channel.execute_wait('ps -p {} > /dev/null 2> /dev/null; echo "STATUS:$?" '.format(
-                    self.resources[job_id]['remote_pid']), self.cmd_timeout)
-                for line in stdout.split('\n'):
-                    if line.startswith("STATUS:"):
-                        status = line.split("STATUS:")[1].strip()
-                        if status == "0":
-                            self.resources[job_id]['status'] = JobStatus(JobState.RUNNING)
-                        else:
-                            self.resources[job_id]['status'] = JobStatus(JobState.FAILED)
+
+            retcode, stdout, stderr = self.channel.execute_wait('ps -p {} > /dev/null 2> /dev/null; echo "STATUS:$?" '.format(
+                self.resources[job_id]['remote_pid']), self.cmd_timeout)
+            for line in stdout.split('\n'):
+                if line.startswith("STATUS:"):
+                    status = line.split("STATUS:")[1].strip()
+                    if status == "0":
+                        self.resources[job_id]['status'] = JobStatus(JobState.RUNNING)
+                    else:
+                        self.resources[job_id]['status'] = JobStatus(JobState.FAILED)
 
         return [self.resources[jid]['status'] for jid in job_ids]
 
@@ -174,34 +157,24 @@ def submit(self, command, tasks_per_node, job_name="parsl.localprovider"):
         self._write_submit_script(wrap_command, script_path)
 
         job_id = None
-        proc = None
         remote_pid = None
         if (self.move_files is None and not isinstance(self.channel, LocalChannel)) or (self.move_files):
             logger.debug("Pushing start script")
             script_path = self.channel.push_file(script_path, self.channel.script_dir)
 
-        if not isinstance(self.channel, LocalChannel):
-            logger.debug("Launching in remote mode")
-            # Bash would return until the streams are closed. So we redirect to a outs file
-            cmd = 'bash {0} > {0}.out 2>&1 & \n echo "PID:$!" '.format(script_path)
-            retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)
-            for line in stdout.split('\n'):
-                if line.startswith("PID:"):
-                    remote_pid = line.split("PID:")[1].strip()
-                    job_id = remote_pid
-            if job_id is None:
-                logger.warning("Channel failed to start remote command/retrieve PID")
-        else:
-
-            try:
-                job_id, proc = self.channel.execute_no_wait('bash {0}'.format(script_path), self.cmd_timeout)
-            except Exception as e:
-                logger.debug("Channel execute failed for: {}, {}".format(self.channel, e))
-                raise
+        logger.debug("Launching in remote mode")
+        # Bash would return until the streams are closed. So we redirect to a outs file
+        cmd = 'bash {0} > {0}.out 2>&1 & \n echo "PID:$!" '.format(script_path)
+        retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)
+        for line in stdout.split('\n'):
+            if line.startswith("PID:"):
+                remote_pid = line.split("PID:")[1].strip()
+                job_id = remote_pid
+        if job_id is None:
+            logger.warning("Channel failed to start remote command/retrieve PID")
 
         self.resources[job_id] = {'job_id': job_id, 'status': JobStatus(JobState.RUNNING),
-                                  'remote_pid': remote_pid,
-                                  'proc': proc}
+                                  'remote_pid': remote_pid}
 
         return job_id
 
@@ -216,18 +189,11 @@ def cancel(self, job_ids):
         '''
         for job in job_ids:
             logger.debug("Terminating job/proc_id: {0}".format(job))
-            # Here we are assuming that for local, the job_ids are the process id's
-            if self.resources[job]['proc']:
-                proc = self.resources[job]['proc']
-                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
-                self.resources[job]['status'] = JobStatus(JobState.CANCELLED)
-
-            elif self.resources[job]['remote_pid']:
-                cmd = "kill -- -$(ps -o pgid= {} | grep -o '[0-9]*')".format(self.resources[job]['remote_pid'])
-                retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)
-                if retcode != 0:
-                    logger.warning("Failed to kill PID: {} and child processes on {}".format(self.resources[job]['remote_pid'],
-                                                                                             self.label))
+            cmd = "kill -- -$(ps -o pgid= {} | grep -o '[0-9]*')".format(self.resources[job]['remote_pid'])
+            retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)
+            if retcode != 0:
+                logger.warning("Failed to kill PID: {} and child processes on {}".format(self.resources[job]['remote_pid'],
+                                                                                         self.label))
 
         rets = [True for i in job_ids]
         return rets


Test patch: