Repository: Parsl/parsl

Issue:
Condor submission failures are muted
In the `CondorProvider.submit()` we do not do a return code check and dump stdout/stderr to error logs. So the logs do go into parsl.log but the user sees little indication as to what happened.


Discussion:

Resolving patch:
diff --git a/parsl/channels/local/local.py b/parsl/channels/local/local.py
--- a/parsl/channels/local/local.py
+++ b/parsl/channels/local/local.py
@@ -73,12 +73,8 @@ def execute_wait(self, cmd, walltime=None, envs={}):
             retcode = proc.returncode
 
         except Exception as e:
-            print("Caught exception: {0}".format(e))
-            logger.warn("Execution of command [%s] failed due to \n %s ", cmd, e)
-            # Set retcode to non-zero so that this can be handled in the provider.
-            if retcode == 0:
-                retcode = -1
-            return (retcode, None, None)
+            logger.warn("Execution of command '{}' failed due to \n{}".format(cmd, e))
+            raise
 
         return (retcode, stdout.decode("utf-8"), stderr.decode("utf-8"))
 
@@ -113,7 +109,7 @@ def execute_no_wait(self, cmd, walltime, envs={}):
             pid = proc.pid
 
         except Exception as e:
-            logger.warn("Execution of command [%s] failed due to \n %s ", (cmd, e))
+            logger.warn("Execution of command '{}' failed due to \n{}".format(cmd, e))
             raise
 
         return pid, proc
diff --git a/parsl/providers/condor/condor.py b/parsl/providers/condor/condor.py
--- a/parsl/providers/condor/condor.py
+++ b/parsl/providers/condor/condor.py
@@ -8,6 +8,7 @@
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.condor.template import template_string
 from parsl.providers.cluster_provider import ClusterProvider
+from parsl.providers.error import ScaleOutFailed
 
 logger = logging.getLogger(__name__)
 
@@ -230,8 +231,10 @@ def submit(self, command, blocksize, tasks_per_node, job_name="parsl.auto"):
         channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)
 
         cmd = "condor_submit {0}".format(channel_script_path)
-        retcode, stdout, stderr = super().execute_wait(cmd)
-        logger.debug("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(), stderr.strip())
+        try:
+            retcode, stdout, stderr = super().execute_wait(cmd)
+        except Exception as e:
+            raise ScaleOutFailed(self.label, str(e))
 
         job_id = []
 
@@ -248,7 +251,12 @@ def submit(self, command, blocksize, tasks_per_node, job_name="parsl.auto"):
                     job_id += [cluster + process for process in processes]
 
             self._add_resource(job_id)
-        return job_id[0]
+            return job_id[0]
+        else:
+            message = "Command '{}' failed with return code {}".format(cmd, retcode)
+            message += " and standard output '{}'".format(stdout.strip()) if stdout is not None else ''
+            message += " and standard error '{}'".format(stderr.strip()) if stderr is not None else ''
+            raise ScaleOutFailed(self.label, message)
 
     def cancel(self, job_ids):
         """Cancels the jobs specified by a list of job IDs.
diff --git a/parsl/providers/error.py b/parsl/providers/error.py
--- a/parsl/providers/error.py
+++ b/parsl/providers/error.py
@@ -42,10 +42,8 @@ def __init__(self, provider, reason):
         self.provider = provider
         self.reason = reason
 
-    def __repr__(self):
-        return "Unable to Initialize provider.Provider:{0}, Reason:{1}".format(
-            self.provider, self.reason
-        )
+    def __str__(self):
+        return "Unable to scale out {} provider: {}".format(self.provider, self.reason)
 
 
 class SchedulerMissingArgs(ExecutionProviderException):


Test patch: